diff --git a/docs/commit-log.md b/docs/commit-log.md new file mode 100644 index 00000000..5a656372 --- /dev/null +++ b/docs/commit-log.md @@ -0,0 +1,58 @@ +# Commit log analysis + +See here https://files.slack.com/files-pri/T03ACD12T-F04V4QC6E/2015-05-21_16.14.50.jpg for details. + +We have some data storage (to decide -- one global storage or separate storage for each resource? +One global commit log or separate for each resource?) -- call it DB for simplicity. + +User modifies some data of some resources K1, K2, H. This data is not stored immediately in the DB, +instead it is stored in some separate place and queued for execution (we call this 'Staged Log'). + +The modified data for a resource is represented as a diff in its inputs. So if user adds new resource +and assigns an IP to it, it is represented something like: + +``` +ip: + from: None + to: 10.20.0.2 +``` + +User commands 'apply'. Orchestrator takes the modified resources and applies appropriate actions +in appropriate order that it computes. + +We think that the 'appropriate action' can be inferred from the diff for each resource. So for example +if resource is new and has added IP the action `run` can be inferred because previous state was +`None` and current is something new. If on the other hand previous state was some value `A` and +new state is some value `B` -- the orchestrator decides that the action to be run is `update`. And +if previous state is some `A` and new state is `None` the action will be `remove`. + +The 'appropriate order' taken by orchestrator can be just like the data flow graph initially. We +see possibility of optimizing the number of actions taken by orchestrator so that moving Keystone +service to another node can be simplified from 4 actions (update HAProxy without removed Keystone, +install Keystone on new node, update HAProxy with new Keystone, remove Keystone from old node) +taken to 3 actions (add Keystone to new node, update HAProxy removing old Keystone and adding +new one, remove Keystone from old node). + +After resource action is finished the new state is saved to the commit log and data is updated in +the DB. + +We want to support rollbacks via commit log. Rollback is done by replaying the commit log backwards. + +In case of separate commit logs per resource we think rollback could be done like this: some resource +`K` is rolled back by one commit log, the diff action is the same as reversed diff action of the +commit we are rolling back. We can update other resources with this new data by analyzing the connections. +So in other words -- we change the data in one resource according to what is in the commit to be rolled +back and then we trigger changes in other connected resources. Then we run orchestrator actions like +described above. + +In case of single commit log for all resources -- is it sufficient to just rollback a commit? Or +do we need to trigger changes in connected resources too? In global commit log we have ordering +of commits like they were run by orchestrator. + +From analysis of resource removal we think that we need to save connection data in each commit -- +otherwise when we rollback that resource removal we wouldn't know how to restore its connections +to other resources. + +Single commits after every action finished on a resource causes many commits per one user 'apply' +action. In order to allow user to revert the whole action and not just single commits we have some +idea of 'tagging' group of commits by some action id. diff --git a/example.py b/example.py index 24a0a756..06903f4c 100644 --- a/example.py +++ b/example.py @@ -1,5 +1,6 @@ import shutil import os +import time from solar.core import resource from solar.core import signals @@ -13,17 +14,16 @@ os.mkdir('rs') node1 = resource.create('node1', 'resources/ro_node/', 'rs/', {'ip':'10.0.0.3', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) node2 = resource.create('node2', 'resources/ro_node/', 'rs/', {'ip':'10.0.0.4', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) -node3 = resource.create('node3', 'resources/ro_node/', 'rs/', {'ip':'10.0.0.5', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) mariadb_service1 = resource.create('mariadb_service1', 'resources/mariadb_service', 'rs/', {'image':'mariadb', 'root_password' : 'mariadb', 'port' : '3306', 'ip': '', 'ssh_user': '', 'ssh_key': ''}) keystone_db = resource.create('keystone_db', 'resources/mariadb_db/', 'rs/', {'db_name':'keystone_db', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''}) keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_user/', 'rs/', {'new_user_name' : 'keystone', 'new_user_password' : 'keystone', 'db_name':'', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''}) keystone_config1 = resource.create('keystone_config1', 'resources/keystone_config/', 'rs/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''}) -keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', 'rs/', {'port':'5000', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) +keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', 'rs/', {'port':'5001', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) keystone_config2 = resource.create('keystone_config2', 'resources/keystone_config/', 'rs/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''}) -keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', 'rs/', {'port':'5000', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) +keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', 'rs/', {'port':'5002', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'resources/haproxy_config/', 'rs/', {'name':'keystone_config', 'listen_port':'5000', 'servers':[], 'ports':[]}) @@ -62,11 +62,12 @@ signals.connect(node2, keystone_service2) signals.connect(keystone_config2, keystone_service2, {'config_dir': 'config_dir'}) signals.connect(keystone_service1, haproxy_keystone_config, {'ip':'servers', 'port':'ports'}) +signals.connect(keystone_service2, haproxy_keystone_config, {'ip':'servers', 'port':'ports'}) -signals.connect(node1, haproxy_config) +signals.connect(node2, haproxy_config) signals.connect(haproxy_keystone_config, haproxy_config, {'listen_port': 'listen_ports', 'name':'configs_names', 'ports' : 'configs_ports', 'servers':'configs'}) -signals.connect(node1, haproxy_service) +signals.connect(node2, haproxy_service) signals.connect(haproxy_config, haproxy_service, {'listen_ports':'ports', 'config_dir':'host_binds'}) @@ -74,19 +75,21 @@ signals.connect(haproxy_config, haproxy_service, {'listen_ports':'ports', 'confi from solar.core import actions actions.resource_action(mariadb_service1, 'run') +time.sleep(10) actions.resource_action(keystone_db, 'run') actions.resource_action(keystone_db_user, 'run') actions.resource_action(keystone_config1, 'run') actions.resource_action(keystone_service1, 'run') +actions.resource_action(keystone_service2, 'run') actions.resource_action(haproxy_config, 'run') actions.resource_action(haproxy_service, 'run') #remove -actions.resource_action(haproxy_service, 'remove') -actions.resource_action(haproxy_config, 'remove') -actions.resource_action(keystone_service1, 'remove') -actions.resource_action(keystone_config1, 'remove') -actions.resource_action(keystone_db_user, 'remove') -actions.resource_action(keystone_db, 'remove') -actions.resource_action(mariadb_service1, 'remove') +#actions.resource_action(haproxy_service, 'remove') +#actions.resource_action(haproxy_config, 'remove') +#actions.resource_action(keystone_service1, 'remove') +#actions.resource_action(keystone_config1, 'remove') +#actions.resource_action(keystone_db_user, 'remove') +#actions.resource_action(keystone_db, 'remove') +#actions.resource_action(mariadb_service1, 'remove') diff --git a/requirements.txt b/requirements.txt index ca7d2def..9b690cee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ click==4.0 jinja2==2.7.3 networkx==1.9.1 PyYAML==3.11 +jsonschema==2.4.0 diff --git a/resources/data_container/meta.yaml b/resources/data_container/meta.yaml index d4185a1a..9910b184 100644 --- a/resources/data_container/meta.yaml +++ b/resources/data_container/meta.yaml @@ -3,5 +3,11 @@ handler: ansible version: 1.0.0 input: ip: + type: str! + value: image: - export_volumes: + type: str! + value: + export_volumes: + type: str! + value: diff --git a/resources/docker_container/meta.yaml b/resources/docker_container/meta.yaml index 182c872c..dd54506e 100644 --- a/resources/docker_container/meta.yaml +++ b/resources/docker_container/meta.yaml @@ -3,13 +3,23 @@ handler: ansible version: 1.0.0 input: ip: + schema: str! + value: image: + schema: str! + value: ports: + schema: [int] + value: [] host_binds: + schema: [int] + value: [] volume_binds: + schema: [int] + value: [] ssh_user: + schema: str! + value: [] ssh_key: -input-types: - ports: - host_binds: list - volume_binds: list + schema: str! + value: [] diff --git a/resources/file/meta.yaml b/resources/file/meta.yaml index 14eb2e2c..19744ba2 100644 --- a/resources/file/meta.yaml +++ b/resources/file/meta.yaml @@ -2,4 +2,6 @@ id: file handler: shell version: 1.0.0 input: - path: /tmp/test_file + path: + schema: str! + value: /tmp/test_file diff --git a/resources/haproxy/meta.yaml b/resources/haproxy/meta.yaml index 57f53a69..793f84d8 100644 --- a/resources/haproxy/meta.yaml +++ b/resources/haproxy/meta.yaml @@ -3,15 +3,26 @@ handler: ansible version: 1.0.0 input: ip: - config_dir: {src: /etc/solar/haproxy, dst: /etc/haproxy} + schema: int! + value: + config_dir: + schema: {src: str!, dst: str!} + value: {src: /etc/solar/haproxy, dst: /etc/haproxy} listen_ports: + schema: [int] + value: [] configs: + schema: [[str]] + value: [] configs_names: + schema: [str] + value: [] configs_ports: + schema: [[int]] + value: [] ssh_user: + schema: str! + value: ssh_key: -input-types: - listen_ports: list - configs: list - configs_names: list - configs_ports: list + schema: str! + value: diff --git a/resources/haproxy_config/meta.yaml b/resources/haproxy_config/meta.yaml index a7584600..307b5345 100644 --- a/resources/haproxy_config/meta.yaml +++ b/resources/haproxy_config/meta.yaml @@ -3,9 +3,14 @@ handler: none version: 1.0.0 input: name: + schema: str! + value: listen_port: + schema: int! + value: ports: + schema: [int] + value: servers: -input-types: - ports: list - servers: list + schema: [str] + value: diff --git a/resources/keystone_config/meta.yaml b/resources/keystone_config/meta.yaml index b4ea7ce6..5124d5dd 100644 --- a/resources/keystone_config/meta.yaml +++ b/resources/keystone_config/meta.yaml @@ -3,11 +3,29 @@ handler: ansible version: 1.0.0 input: config_dir: + schema: str! + value: admin_token: + schema: str! + value: db_user: + schema: str! + value: db_password: + schema: str! + value: db_host: + schema: str! + value: db_name: + schema: str! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/resources/keystone_service/meta.yaml b/resources/keystone_service/meta.yaml index 1e3add8d..e05179dc 100644 --- a/resources/keystone_service/meta.yaml +++ b/resources/keystone_service/meta.yaml @@ -2,10 +2,24 @@ id: keystone handler: ansible version: 1.0.0 input: - image: kollaglue/centos-rdo-keystone + image: + schema: str! + value: kollaglue/centos-rdo-keystone config_dir: + schema: str! + value: port: + schema: int! + value: admin_port: + schema: int! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/resources/keystone_user/meta.yaml b/resources/keystone_user/meta.yaml index 6293937c..0057019f 100644 --- a/resources/keystone_user/meta.yaml +++ b/resources/keystone_user/meta.yaml @@ -3,12 +3,32 @@ handler: ansible version: 1.0.0 input: keystone_host: + schema: str! + value: keystone_port: + schema: int! + value: login_user: + schema: str! + value: login_token: + schema: str! + value: user_name: + schema: str! + value: user_password: + schema: str! + value: tenant_name: + schema: str! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/resources/mariadb_db/meta.yaml b/resources/mariadb_db/meta.yaml index 609814ca..41b952e6 100644 --- a/resources/mariadb_db/meta.yaml +++ b/resources/mariadb_db/meta.yaml @@ -6,9 +6,23 @@ actions: remove: remove.yml input: db_name: + schema: str! + value: login_password: + schema: str! + value: login_port: + schema: int! + value: login_user: - ip: - ssh_key: - ssh_user: + schema: str! + value: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/resources/mariadb_service/meta.yaml b/resources/mariadb_service/meta.yaml index 6fa200d0..4295b7e1 100644 --- a/resources/mariadb_service/meta.yaml +++ b/resources/mariadb_service/meta.yaml @@ -3,8 +3,20 @@ handler: ansible version: 1.0.0 input: image: - root_password: - port: - ip: - ssh_key: - ssh_user: + schema: str! + value: + root_password: + schema: str! + value: + port: + schema: str! + value: + ip: + schema: int! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/resources/mariadb_user/meta.yaml b/resources/mariadb_user/meta.yaml index b45f8e47..b46d6e26 100644 --- a/resources/mariadb_user/meta.yaml +++ b/resources/mariadb_user/meta.yaml @@ -6,11 +6,29 @@ actions: remove: remove.yml input: new_user_password: + schema: str! + value: new_user_name: + schema: str! + value: db_name: + schema: str! + value: login_password: + schema: str! + value: login_port: + schema: int! + value: login_user: - ip: - ssh_key: - ssh_user: + schema: str! + value: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/resources/nova/meta.yaml b/resources/nova/meta.yaml index 0591a410..5c48b0b9 100644 --- a/resources/nova/meta.yaml +++ b/resources/nova/meta.yaml @@ -3,5 +3,11 @@ handler: ansible version: 1.0.0 input: ip: - port: 8774 + schema: str! + value: + port: + schema: int! + value: 8774 image: # TODO + schema: str! + value: diff --git a/resources/ro_node/meta.yaml b/resources/ro_node/meta.yaml index 1ceaa0fc..c37fcb7f 100644 --- a/resources/ro_node/meta.yaml +++ b/resources/ro_node/meta.yaml @@ -4,5 +4,11 @@ version: 1.0.0 actions: input: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/run_tests.sh b/run_tests.sh index 6dffeda3..6f3572f5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,8 +17,9 @@ fi pip install -r requirements.txt --download-cache=/tmp/$JOB_NAME -pushd x +pushd solar/solar -PYTHONPATH=$WORKSPACE CONFIG_FILE=$CONFIG_FILE python test/test_signals.py +PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE python test/test_signals.py +PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE python test/test_validation.py popd diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index fbfe007e..4f28651a 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -13,6 +13,7 @@ from solar.core import db from solar.core import observer from solar.core import signals from solar.core import utils +from solar.core import validation class Resource(object): @@ -21,14 +22,12 @@ class Resource(object): self.base_dir = base_dir self.metadata = metadata self.actions = metadata['actions'].keys() if metadata['actions'] else None - self.requires = metadata['input'].keys() - self._validate_args(args, metadata['input']) self.args = {} for arg_name, arg_value in args.items(): - type_ = metadata.get('input-types', {}).get(arg_name) or 'simple' + metadata_arg = self.metadata['input'][arg_name] + type_ = validation.schema_input_type(metadata_arg.get('schema', 'str')) + self.args[arg_name] = observer.create(type_, self, arg_name, arg_value) - self.metadata['input'] = args - self.input_types = metadata.get('input-types', {}) self.changed = [] self.tags = tags or [] @@ -95,22 +94,13 @@ class Resource(object): else: raise Exception('Uuups, action is not available') - def _validate_args(self, args, inputs): - for req in self.requires: - if req not in args: - # If metadata input is filled with a value, use it as default - # and don't report an error - if inputs.get(req): - args[req] = inputs[req] - else: - raise Exception('Requirement `{0}` is missing in args'.format(req)) - # TODO: versioning def save(self): metadata = copy.deepcopy(self.metadata) metadata['tags'] = self.tags - metadata['input'] = self.args_dict() + for k, v in self.args_dict().items(): + metadata['input'][k]['value'] = v meta_file = os.path.join(self.base_dir, 'meta.yaml') with open(meta_file, 'w') as f: diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 05ca425a..9bcc0b3d 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -83,8 +83,8 @@ def guess_mapping(emitter, receiver): :return: """ guessed = {} - for key in emitter.requires: - if key in receiver.requires: + for key in emitter.args: + if key in receiver.args: guessed[key] = key return guessed diff --git a/solar/solar/core/validation.py b/solar/solar/core/validation.py new file mode 100644 index 00000000..01045f8a --- /dev/null +++ b/solar/solar/core/validation.py @@ -0,0 +1,115 @@ +from jsonschema import validate, ValidationError, SchemaError + + +def schema_input_type(schema): + """Input type from schema + + :param schema: + :return: simple/list + """ + if isinstance(schema, list): + return 'list' + + return 'simple' + + +def _construct_jsonschema(schema, definition_base=''): + """Construct jsonschema from our metadata input schema. + + :param schema: + :return: + """ + if schema == 'str': + return {'type': 'string'}, {} + + if schema == 'str!': + return {'type': 'string', 'minLength': 1}, {} + + if schema == 'int' or schema == 'int!': + return {'type': 'number'}, {} + + if isinstance(schema, list): + items, definitions = _construct_jsonschema(schema[0], definition_base=definition_base) + + return { + 'type': 'array', + 'items': items, + }, definitions + + if isinstance(schema, dict): + properties = {} + definitions = {} + + for k, v in schema.items(): + if isinstance(v, dict) or isinstance(v, list): + key = '{}_{}'.format(definition_base, k) + properties[k] = {'$ref': '#/definitions/{}'.format(key)} + definitions[key], new_definitions = _construct_jsonschema(v, definition_base=key) + else: + properties[k], new_definitions = _construct_jsonschema(v, definition_base=definition_base) + + definitions.update(new_definitions) + + required = [k for k, v in schema.items() if + isinstance(v, basestring) and v.endswith('!')] + + ret = { + 'type': 'object', + 'properties': properties, + } + + if required: + ret['required'] = required + + return ret, definitions + + +def construct_jsonschema(schema): + jsonschema, definitions = _construct_jsonschema(schema) + + jsonschema['definitions'] = definitions + + return jsonschema + + +def validate_input(value, jsonschema=None, schema=None): + """Validate single input according to schema. + + :param value: Value to be validated + :param schema: Dict in jsonschema format + :param schema: Our custom, simplified schema + :return: list with errors + """ + if jsonschema is None: + jsonschema = construct_jsonschema(schema) + try: + validate(value, jsonschema) + except ValidationError as e: + return [e.message] + except: + print 'jsonschema', jsonschema + print 'value', value + raise + + +def validate_resource(r): + """Check if resource inputs correspond to schema. + + :param r: Resource instance + :return: dict, keys are input names, value is array with error. + """ + ret = {} + + input_schemas = r.metadata['input'] + args = r.args_dict() + + for input_name, input_definition in input_schemas.items(): + errors = validate_input( + args.get(input_name), + jsonschema=input_definition.get('jsonschema'), + schema=input_definition.get('schema') + ) + if errors: + ret[input_name] = errors + + return ret diff --git a/solar/solar/test/base.py b/solar/solar/test/base.py index f58e727b..7dfcd82c 100644 --- a/solar/solar/test/base.py +++ b/solar/solar/test/base.py @@ -4,9 +4,9 @@ import tempfile import unittest import yaml -from x import db -from x import resource as xr -from x import signals as xs +from solar.core import db +from solar.core import resource as xr +from solar.core import signals as xs class BaseResourceTest(unittest.TestCase): diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index 59887add..8b80e708 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -2,7 +2,7 @@ import unittest import base -from x import signals as xs +from solar.core import signals as xs class TestBaseInput(base.BaseResourceTest): @@ -12,7 +12,9 @@ id: sample handler: ansible version: 1.0.0 input: - values: {} + values: + schema: {a: int, b: int} + value: {} """) sample1 = self.create_resource( @@ -63,7 +65,11 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: port: + schema: int + value: """) sample_ip_meta_dir = self.make_resource_meta(""" id: sample-ip @@ -71,6 +77,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: """) sample_port_meta_dir = self.make_resource_meta(""" id: sample-port @@ -78,6 +86,8 @@ handler: ansible version: 1.0.0 input: port: + schema: int + value: """) sample = self.create_resource( @@ -109,6 +119,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: """) sample = self.create_resource( @@ -149,6 +161,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: """) sample1 = self.create_resource( @@ -171,6 +185,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: """) list_input_single_meta_dir = self.make_resource_meta(""" id: list-input-single @@ -178,8 +194,8 @@ handler: ansible version: 1.0.0 input: ips: -input-types: - ips: list + schema: [str] + value: [] """) sample1 = self.create_resource( @@ -248,7 +264,11 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: port: + schema: int + value: """) list_input_multi_meta_dir = self.make_resource_meta(""" id: list-input-multi @@ -256,10 +276,11 @@ handler: ansible version: 1.0.0 input: ips: + schema: [str] + value: ports: -input-types: - ips: list - ports: list + schema: [int] + value: """) sample1 = self.create_resource( diff --git a/solar/solar/test/test_validation.py b/solar/solar/test/test_validation.py new file mode 100644 index 00000000..da0da6d0 --- /dev/null +++ b/solar/solar/test/test_validation.py @@ -0,0 +1,174 @@ +import unittest + +from solar.test import base + +from solar.core import validation as sv + + +class TestInputValidation(base.BaseResourceTest): + def test_input_str_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + value: + schema: str + value: + value-required: + schema: str! + value: + """) + + r = self.create_resource( + 'r1', sample_meta_dir, {'value': 'x', 'value-required': 'y'} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r2', sample_meta_dir, {'value': 1, 'value-required': 'y'} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value']) + + r = self.create_resource( + 'r3', sample_meta_dir, {'value': ''} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value-required']) + + + def test_input_int_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + value: + schema: int + value: + value-required: + schema: int! + value: + """) + + r = self.create_resource( + 'r1', sample_meta_dir, {'value': 1, 'value-required': 2} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r2', sample_meta_dir, {'value': 'x', 'value-required': 2} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value']) + + r = self.create_resource( + 'r3', sample_meta_dir, {'value': 1} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value-required']) + + def test_input_dict_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {a: int!, b: int} + value: {} + """) + + r = self.create_resource( + 'r', sample_meta_dir, {'values': {'a': 1, 'b': 2}} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({'values': None}) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'a': 1, 'c': 3}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r1', sample_meta_dir, {'values': {'b': 2}} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + def test_complex_input(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {l: [{a: int}]} + value: {l: [{a: 1}]} + """) + + r = self.create_resource( + 'r', sample_meta_dir, { + 'values': { + 'l': [{'a': 1}], + } + } + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({ + 'values': { + 'l': [{'a': 'x'}], + } + }) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'l': [{'a': 1, 'c': 3}]}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + def test_more_complex_input(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {l: [{a: int}], d: {x: [int]}} + value: {l: [{a: 1}], d: {x: [1, 2]}} + """) + + r = self.create_resource( + 'r', sample_meta_dir, { + 'values': { + 'l': [{'a': 1}], + 'd': {'x': [1, 2]} + } + } + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({ + 'values': { + 'l': [{'a': 1}], + 'd': [] + } + }) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'a': 1, 'c': 3}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + +if __name__ == '__main__': + unittest.main() diff --git a/x-TODO.md b/x-TODO.md index f52e48a1..0263054f 100644 --- a/x-TODO.md +++ b/x-TODO.md @@ -1,5 +1,7 @@ # TODO +- grammar connections fuzzy matching algorithm (for example: type 'login' joins to type 'login' irrespective of names of both inputs) +- resource connections JS frontend (?) - store all resource configurations somewhere globally (this is required to correctly perform an update on one resource and bubble down to all others) - config templates @@ -9,6 +11,7 @@ when some image is unused to conserve space # DONE +- CI - Deploy HAProxy, Keystone and MariaDB - ansible handler (loles) - tags are kept in resource mata file (pkaminski)