From e86ac3f8378f076421353fb161235b9131da247b Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 14 May 2015 17:45:40 +0200 Subject: [PATCH 1/6] Added resource validation - uses jsonschema --- requirements.txt | 1 + run_tests.sh | 5 +- solar/solar/core/resource.py | 22 ++--- solar/solar/core/signals.py | 4 +- solar/solar/core/validation.py | 86 ++++++++++++++++++++ solar/solar/test/base.py | 6 +- solar/solar/test/test_signals.py | 35 ++++++-- solar/solar/test/test_validation.py | 106 +++++++++++++++++++++++++ x/TODO.md | 3 + x/resources/data_container/meta.yaml | 8 +- x/resources/docker_container/meta.yaml | 18 ++++- x/resources/file/meta.yaml | 4 +- x/resources/haproxy/meta.yaml | 23 ++++-- x/resources/haproxy_config/meta.yaml | 11 ++- x/resources/keystone_config/meta.yaml | 18 +++++ x/resources/keystone_service/meta.yaml | 16 +++- x/resources/keystone_user/meta.yaml | 20 +++++ x/resources/mariadb_db/meta.yaml | 20 ++++- x/resources/mariadb_service/meta.yaml | 22 +++-- x/resources/mariadb_user/meta.yaml | 24 +++++- x/resources/nova/meta.yaml | 8 +- x/resources/ro_node/meta.yaml | 6 ++ 22 files changed, 408 insertions(+), 58 deletions(-) create mode 100644 solar/solar/core/validation.py create mode 100644 solar/solar/test/test_validation.py diff --git a/requirements.txt b/requirements.txt index ca7d2def..9b690cee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ click==4.0 jinja2==2.7.3 networkx==1.9.1 PyYAML==3.11 +jsonschema==2.4.0 diff --git a/run_tests.sh b/run_tests.sh index 6dffeda3..6f3572f5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,8 +17,9 @@ fi pip install -r requirements.txt --download-cache=/tmp/$JOB_NAME -pushd x +pushd solar/solar -PYTHONPATH=$WORKSPACE CONFIG_FILE=$CONFIG_FILE python test/test_signals.py +PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE python test/test_signals.py +PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE python test/test_validation.py popd diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index fbfe007e..4f28651a 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -13,6 +13,7 @@ from solar.core import db from solar.core import observer from solar.core import signals from solar.core import utils +from solar.core import validation class Resource(object): @@ -21,14 +22,12 @@ class Resource(object): self.base_dir = base_dir self.metadata = metadata self.actions = metadata['actions'].keys() if metadata['actions'] else None - self.requires = metadata['input'].keys() - self._validate_args(args, metadata['input']) self.args = {} for arg_name, arg_value in args.items(): - type_ = metadata.get('input-types', {}).get(arg_name) or 'simple' + metadata_arg = self.metadata['input'][arg_name] + type_ = validation.schema_input_type(metadata_arg.get('schema', 'str')) + self.args[arg_name] = observer.create(type_, self, arg_name, arg_value) - self.metadata['input'] = args - self.input_types = metadata.get('input-types', {}) self.changed = [] self.tags = tags or [] @@ -95,22 +94,13 @@ class Resource(object): else: raise Exception('Uuups, action is not available') - def _validate_args(self, args, inputs): - for req in self.requires: - if req not in args: - # If metadata input is filled with a value, use it as default - # and don't report an error - if inputs.get(req): - args[req] = inputs[req] - else: - raise Exception('Requirement `{0}` is missing in args'.format(req)) - # TODO: versioning def save(self): metadata = copy.deepcopy(self.metadata) metadata['tags'] = self.tags - metadata['input'] = self.args_dict() + for k, v in self.args_dict().items(): + metadata['input'][k]['value'] = v meta_file = os.path.join(self.base_dir, 'meta.yaml') with open(meta_file, 'w') as f: diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 05ca425a..9bcc0b3d 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -83,8 +83,8 @@ def guess_mapping(emitter, receiver): :return: """ guessed = {} - for key in emitter.requires: - if key in receiver.requires: + for key in emitter.args: + if key in receiver.args: guessed[key] = key return guessed diff --git a/solar/solar/core/validation.py b/solar/solar/core/validation.py new file mode 100644 index 00000000..623a950e --- /dev/null +++ b/solar/solar/core/validation.py @@ -0,0 +1,86 @@ +from jsonschema import validate, ValidationError + + +def schema_input_type(schema): + """Input type from schema + + :param schema: + :return: simple/list + """ + if isinstance(schema, list): + return 'list' + + return 'simple' + + +def construct_jsonschema(schema): + """Construct jsonschema from our metadata input schema. + + :param schema: + :return: + """ + + if schema == 'str': + return {'type': 'string'} + + if schema == 'str!': + return {'type': 'string', 'minLength': 1} + + if schema == 'int' or schema == 'int!': + return {'type': 'number'} + + if isinstance(schema, list): + return { + 'type': 'array', + 'items': construct_jsonschema(schema[0]), + } + + if isinstance(schema, dict): + return { + 'type': 'object', + 'properties': { + k: construct_jsonschema(v) for k, v in schema.items() + }, + 'required': [k for k, v in schema.items() if + isinstance(v, basestring) and v.endswith('!')], + } + + +def validate_input(value, jsonschema=None, schema=None): + """Validate single input according to schema. + + :param value: Value to be validated + :param schema: Dict in jsonschema format + :param schema: Our custom, simplified schema + :return: list with errors + """ + try: + if jsonschema: + validate(value, jsonschema) + else: + validate(value, construct_jsonschema(schema)) + except ValidationError as e: + return [e.message] + + +def validate_resource(r): + """Check if resource inputs correspond to schema. + + :param r: Resource instance + :return: dict, keys are input names, value is array with error. + """ + ret = {} + + input_schemas = r.metadata['input'] + args = r.args_dict() + + for input_name, input_definition in input_schemas.items(): + errors = validate_input( + args.get(input_name), + jsonschema=input_definition.get('jsonschema'), + schema=input_definition.get('schema') + ) + if errors: + ret[input_name] = errors + + return ret diff --git a/solar/solar/test/base.py b/solar/solar/test/base.py index f58e727b..7dfcd82c 100644 --- a/solar/solar/test/base.py +++ b/solar/solar/test/base.py @@ -4,9 +4,9 @@ import tempfile import unittest import yaml -from x import db -from x import resource as xr -from x import signals as xs +from solar.core import db +from solar.core import resource as xr +from solar.core import signals as xs class BaseResourceTest(unittest.TestCase): diff --git a/solar/solar/test/test_signals.py b/solar/solar/test/test_signals.py index 59887add..8b80e708 100644 --- a/solar/solar/test/test_signals.py +++ b/solar/solar/test/test_signals.py @@ -2,7 +2,7 @@ import unittest import base -from x import signals as xs +from solar.core import signals as xs class TestBaseInput(base.BaseResourceTest): @@ -12,7 +12,9 @@ id: sample handler: ansible version: 1.0.0 input: - values: {} + values: + schema: {a: int, b: int} + value: {} """) sample1 = self.create_resource( @@ -63,7 +65,11 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: port: + schema: int + value: """) sample_ip_meta_dir = self.make_resource_meta(""" id: sample-ip @@ -71,6 +77,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: """) sample_port_meta_dir = self.make_resource_meta(""" id: sample-port @@ -78,6 +86,8 @@ handler: ansible version: 1.0.0 input: port: + schema: int + value: """) sample = self.create_resource( @@ -109,6 +119,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: string + value: """) sample = self.create_resource( @@ -149,6 +161,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: """) sample1 = self.create_resource( @@ -171,6 +185,8 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: """) list_input_single_meta_dir = self.make_resource_meta(""" id: list-input-single @@ -178,8 +194,8 @@ handler: ansible version: 1.0.0 input: ips: -input-types: - ips: list + schema: [str] + value: [] """) sample1 = self.create_resource( @@ -248,7 +264,11 @@ handler: ansible version: 1.0.0 input: ip: + schema: str + value: port: + schema: int + value: """) list_input_multi_meta_dir = self.make_resource_meta(""" id: list-input-multi @@ -256,10 +276,11 @@ handler: ansible version: 1.0.0 input: ips: + schema: [str] + value: ports: -input-types: - ips: list - ports: list + schema: [int] + value: """) sample1 = self.create_resource( diff --git a/solar/solar/test/test_validation.py b/solar/solar/test/test_validation.py new file mode 100644 index 00000000..d0cce15b --- /dev/null +++ b/solar/solar/test/test_validation.py @@ -0,0 +1,106 @@ +import unittest + +import base + +from solar.core import validation as sv + + +class TestInputValidation(base.BaseResourceTest): + def test_input_str_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + value: + schema: str + value: + value-required: + schema: str! + value: + """) + + r = self.create_resource( + 'r1', sample_meta_dir, {'value': 'x', 'value-required': 'y'} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r2', sample_meta_dir, {'value': 1, 'value-required': 'y'} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value']) + + r = self.create_resource( + 'r3', sample_meta_dir, {'value': ''} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value-required']) + + def test_input_int_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + value: + schema: int + value: + value-required: + schema: int! + value: + """) + + r = self.create_resource( + 'r1', sample_meta_dir, {'value': 1, 'value-required': 2} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r2', sample_meta_dir, {'value': 'x', 'value-required': 2} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value']) + + r = self.create_resource( + 'r3', sample_meta_dir, {'value': 1} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['value-required']) + + def test_input_dict_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {a: int!, b: int} + value: {} + """) + + r = self.create_resource( + 'r', sample_meta_dir, {'values': {'a': 1, 'b': 2}} + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({'values': None}) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'a': 1, 'c': 3}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r = self.create_resource( + 'r1', sample_meta_dir, {'values': {'b': 2}} + ) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + +if __name__ == '__main__': + unittest.main() diff --git a/x/TODO.md b/x/TODO.md index f52e48a1..0263054f 100644 --- a/x/TODO.md +++ b/x/TODO.md @@ -1,5 +1,7 @@ # TODO +- grammar connections fuzzy matching algorithm (for example: type 'login' joins to type 'login' irrespective of names of both inputs) +- resource connections JS frontend (?) - store all resource configurations somewhere globally (this is required to correctly perform an update on one resource and bubble down to all others) - config templates @@ -9,6 +11,7 @@ when some image is unused to conserve space # DONE +- CI - Deploy HAProxy, Keystone and MariaDB - ansible handler (loles) - tags are kept in resource mata file (pkaminski) diff --git a/x/resources/data_container/meta.yaml b/x/resources/data_container/meta.yaml index d4185a1a..9910b184 100644 --- a/x/resources/data_container/meta.yaml +++ b/x/resources/data_container/meta.yaml @@ -3,5 +3,11 @@ handler: ansible version: 1.0.0 input: ip: + type: str! + value: image: - export_volumes: + type: str! + value: + export_volumes: + type: str! + value: diff --git a/x/resources/docker_container/meta.yaml b/x/resources/docker_container/meta.yaml index 182c872c..dd54506e 100644 --- a/x/resources/docker_container/meta.yaml +++ b/x/resources/docker_container/meta.yaml @@ -3,13 +3,23 @@ handler: ansible version: 1.0.0 input: ip: + schema: str! + value: image: + schema: str! + value: ports: + schema: [int] + value: [] host_binds: + schema: [int] + value: [] volume_binds: + schema: [int] + value: [] ssh_user: + schema: str! + value: [] ssh_key: -input-types: - ports: - host_binds: list - volume_binds: list + schema: str! + value: [] diff --git a/x/resources/file/meta.yaml b/x/resources/file/meta.yaml index 14eb2e2c..19744ba2 100644 --- a/x/resources/file/meta.yaml +++ b/x/resources/file/meta.yaml @@ -2,4 +2,6 @@ id: file handler: shell version: 1.0.0 input: - path: /tmp/test_file + path: + schema: str! + value: /tmp/test_file diff --git a/x/resources/haproxy/meta.yaml b/x/resources/haproxy/meta.yaml index 57f53a69..793f84d8 100644 --- a/x/resources/haproxy/meta.yaml +++ b/x/resources/haproxy/meta.yaml @@ -3,15 +3,26 @@ handler: ansible version: 1.0.0 input: ip: - config_dir: {src: /etc/solar/haproxy, dst: /etc/haproxy} + schema: int! + value: + config_dir: + schema: {src: str!, dst: str!} + value: {src: /etc/solar/haproxy, dst: /etc/haproxy} listen_ports: + schema: [int] + value: [] configs: + schema: [[str]] + value: [] configs_names: + schema: [str] + value: [] configs_ports: + schema: [[int]] + value: [] ssh_user: + schema: str! + value: ssh_key: -input-types: - listen_ports: list - configs: list - configs_names: list - configs_ports: list + schema: str! + value: diff --git a/x/resources/haproxy_config/meta.yaml b/x/resources/haproxy_config/meta.yaml index a7584600..307b5345 100644 --- a/x/resources/haproxy_config/meta.yaml +++ b/x/resources/haproxy_config/meta.yaml @@ -3,9 +3,14 @@ handler: none version: 1.0.0 input: name: + schema: str! + value: listen_port: + schema: int! + value: ports: + schema: [int] + value: servers: -input-types: - ports: list - servers: list + schema: [str] + value: diff --git a/x/resources/keystone_config/meta.yaml b/x/resources/keystone_config/meta.yaml index b4ea7ce6..5124d5dd 100644 --- a/x/resources/keystone_config/meta.yaml +++ b/x/resources/keystone_config/meta.yaml @@ -3,11 +3,29 @@ handler: ansible version: 1.0.0 input: config_dir: + schema: str! + value: admin_token: + schema: str! + value: db_user: + schema: str! + value: db_password: + schema: str! + value: db_host: + schema: str! + value: db_name: + schema: str! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/x/resources/keystone_service/meta.yaml b/x/resources/keystone_service/meta.yaml index 1e3add8d..e05179dc 100644 --- a/x/resources/keystone_service/meta.yaml +++ b/x/resources/keystone_service/meta.yaml @@ -2,10 +2,24 @@ id: keystone handler: ansible version: 1.0.0 input: - image: kollaglue/centos-rdo-keystone + image: + schema: str! + value: kollaglue/centos-rdo-keystone config_dir: + schema: str! + value: port: + schema: int! + value: admin_port: + schema: int! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/x/resources/keystone_user/meta.yaml b/x/resources/keystone_user/meta.yaml index 6293937c..0057019f 100644 --- a/x/resources/keystone_user/meta.yaml +++ b/x/resources/keystone_user/meta.yaml @@ -3,12 +3,32 @@ handler: ansible version: 1.0.0 input: keystone_host: + schema: str! + value: keystone_port: + schema: int! + value: login_user: + schema: str! + value: login_token: + schema: str! + value: user_name: + schema: str! + value: user_password: + schema: str! + value: tenant_name: + schema: str! + value: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: diff --git a/x/resources/mariadb_db/meta.yaml b/x/resources/mariadb_db/meta.yaml index 609814ca..41b952e6 100644 --- a/x/resources/mariadb_db/meta.yaml +++ b/x/resources/mariadb_db/meta.yaml @@ -6,9 +6,23 @@ actions: remove: remove.yml input: db_name: + schema: str! + value: login_password: + schema: str! + value: login_port: + schema: int! + value: login_user: - ip: - ssh_key: - ssh_user: + schema: str! + value: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/x/resources/mariadb_service/meta.yaml b/x/resources/mariadb_service/meta.yaml index 6fa200d0..4295b7e1 100644 --- a/x/resources/mariadb_service/meta.yaml +++ b/x/resources/mariadb_service/meta.yaml @@ -3,8 +3,20 @@ handler: ansible version: 1.0.0 input: image: - root_password: - port: - ip: - ssh_key: - ssh_user: + schema: str! + value: + root_password: + schema: str! + value: + port: + schema: str! + value: + ip: + schema: int! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/x/resources/mariadb_user/meta.yaml b/x/resources/mariadb_user/meta.yaml index b45f8e47..b46d6e26 100644 --- a/x/resources/mariadb_user/meta.yaml +++ b/x/resources/mariadb_user/meta.yaml @@ -6,11 +6,29 @@ actions: remove: remove.yml input: new_user_password: + schema: str! + value: new_user_name: + schema: str! + value: db_name: + schema: str! + value: login_password: + schema: str! + value: login_port: + schema: int! + value: login_user: - ip: - ssh_key: - ssh_user: + schema: str! + value: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: diff --git a/x/resources/nova/meta.yaml b/x/resources/nova/meta.yaml index 0591a410..5c48b0b9 100644 --- a/x/resources/nova/meta.yaml +++ b/x/resources/nova/meta.yaml @@ -3,5 +3,11 @@ handler: ansible version: 1.0.0 input: ip: - port: 8774 + schema: str! + value: + port: + schema: int! + value: 8774 image: # TODO + schema: str! + value: diff --git a/x/resources/ro_node/meta.yaml b/x/resources/ro_node/meta.yaml index 1ceaa0fc..c37fcb7f 100644 --- a/x/resources/ro_node/meta.yaml +++ b/x/resources/ro_node/meta.yaml @@ -4,5 +4,11 @@ version: 1.0.0 actions: input: ip: + schema: str! + value: ssh_key: + schema: str! + value: ssh_user: + schema: str! + value: From 665b1d8c371a165b25f96bff2212c41598131ef8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 20 May 2015 12:44:19 +0200 Subject: [PATCH 2/6] Fix test imports --- solar/solar/test/test_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solar/solar/test/test_validation.py b/solar/solar/test/test_validation.py index d0cce15b..0b4afa6d 100644 --- a/solar/solar/test/test_validation.py +++ b/solar/solar/test/test_validation.py @@ -1,6 +1,6 @@ import unittest -import base +from solar.test import base from solar.core import validation as sv @@ -38,6 +38,7 @@ input: errors = sv.validate_resource(r) self.assertListEqual(errors.keys(), ['value-required']) + def test_input_int_type(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -101,6 +102,5 @@ input: errors = sv.validate_resource(r) self.assertListEqual(errors.keys(), ['values']) - if __name__ == '__main__': unittest.main() From 7cdb7f184311eaa5abc73caccd767dce6bd1123e Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 21 May 2015 13:54:57 +0200 Subject: [PATCH 3/6] Fix nested jsonschema construction --- solar/solar/core/validation.py | 65 +++++++++++++++++++-------- solar/solar/test/test_validation.py | 68 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 18 deletions(-) diff --git a/solar/solar/core/validation.py b/solar/solar/core/validation.py index 623a950e..01045f8a 100644 --- a/solar/solar/core/validation.py +++ b/solar/solar/core/validation.py @@ -1,4 +1,4 @@ -from jsonschema import validate, ValidationError +from jsonschema import validate, ValidationError, SchemaError def schema_input_type(schema): @@ -13,38 +13,64 @@ def schema_input_type(schema): return 'simple' -def construct_jsonschema(schema): +def _construct_jsonschema(schema, definition_base=''): """Construct jsonschema from our metadata input schema. :param schema: :return: """ - if schema == 'str': - return {'type': 'string'} + return {'type': 'string'}, {} if schema == 'str!': - return {'type': 'string', 'minLength': 1} + return {'type': 'string', 'minLength': 1}, {} if schema == 'int' or schema == 'int!': - return {'type': 'number'} + return {'type': 'number'}, {} if isinstance(schema, list): + items, definitions = _construct_jsonschema(schema[0], definition_base=definition_base) + return { 'type': 'array', - 'items': construct_jsonschema(schema[0]), - } + 'items': items, + }, definitions if isinstance(schema, dict): - return { + properties = {} + definitions = {} + + for k, v in schema.items(): + if isinstance(v, dict) or isinstance(v, list): + key = '{}_{}'.format(definition_base, k) + properties[k] = {'$ref': '#/definitions/{}'.format(key)} + definitions[key], new_definitions = _construct_jsonschema(v, definition_base=key) + else: + properties[k], new_definitions = _construct_jsonschema(v, definition_base=definition_base) + + definitions.update(new_definitions) + + required = [k for k, v in schema.items() if + isinstance(v, basestring) and v.endswith('!')] + + ret = { 'type': 'object', - 'properties': { - k: construct_jsonschema(v) for k, v in schema.items() - }, - 'required': [k for k, v in schema.items() if - isinstance(v, basestring) and v.endswith('!')], + 'properties': properties, } + if required: + ret['required'] = required + + return ret, definitions + + +def construct_jsonschema(schema): + jsonschema, definitions = _construct_jsonschema(schema) + + jsonschema['definitions'] = definitions + + return jsonschema + def validate_input(value, jsonschema=None, schema=None): """Validate single input according to schema. @@ -54,13 +80,16 @@ def validate_input(value, jsonschema=None, schema=None): :param schema: Our custom, simplified schema :return: list with errors """ + if jsonschema is None: + jsonschema = construct_jsonschema(schema) try: - if jsonschema: - validate(value, jsonschema) - else: - validate(value, construct_jsonschema(schema)) + validate(value, jsonschema) except ValidationError as e: return [e.message] + except: + print 'jsonschema', jsonschema + print 'value', value + raise def validate_resource(r): diff --git a/solar/solar/test/test_validation.py b/solar/solar/test/test_validation.py index 0b4afa6d..da0da6d0 100644 --- a/solar/solar/test/test_validation.py +++ b/solar/solar/test/test_validation.py @@ -102,5 +102,73 @@ input: errors = sv.validate_resource(r) self.assertListEqual(errors.keys(), ['values']) + def test_complex_input(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {l: [{a: int}]} + value: {l: [{a: 1}]} + """) + + r = self.create_resource( + 'r', sample_meta_dir, { + 'values': { + 'l': [{'a': 1}], + } + } + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({ + 'values': { + 'l': [{'a': 'x'}], + } + }) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'l': [{'a': 1, 'c': 3}]}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + def test_more_complex_input(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: + schema: {l: [{a: int}], d: {x: [int]}} + value: {l: [{a: 1}], d: {x: [1, 2]}} + """) + + r = self.create_resource( + 'r', sample_meta_dir, { + 'values': { + 'l': [{'a': 1}], + 'd': {'x': [1, 2]} + } + } + ) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + + r.update({ + 'values': { + 'l': [{'a': 1}], + 'd': [] + } + }) + errors = sv.validate_resource(r) + self.assertListEqual(errors.keys(), ['values']) + + r.update({'values': {'a': 1, 'c': 3}}) + errors = sv.validate_resource(r) + self.assertEqual(errors, {}) + if __name__ == '__main__': unittest.main() From 558b6684ae33ddb9eb5dcb83d04a71e94c7c0208 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Fri, 22 May 2015 09:45:44 +0200 Subject: [PATCH 4/6] Docs: commit log --- docs/commit-log.md | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/commit-log.md diff --git a/docs/commit-log.md b/docs/commit-log.md new file mode 100644 index 00000000..bfdaf32c --- /dev/null +++ b/docs/commit-log.md @@ -0,0 +1,47 @@ +# Commit log analysis + +See here https://files.slack.com/files-pri/T03ACD12T-F04V4QC6E/2015-05-21_16.14.50.jpg for details. + +We have some data storage (to decide -- one global storage or separate storage for each resource? +One global commit log or separate for each resource?) -- call it DB for simplicity. + +User modifies some data of some resources K1, K2, H. This data is not stored immediately in the DB, +instead it is stored in some separate place and queued for execution (we call this 'Staged Log'). + +The modified data for a resource is represented as a diff in its inputs. So if user adds new resource +and assigns an IP to it, it is represented something like: + +``` +ip: + from: None + to: 10.20.0.2 +``` + +User commands 'apply'. Orchestrator takes the modified resources and applies appropriate actions +in appropriate order that it computes. + +We think that the 'appropriate action' can be inferred from the diff for each resource. So for example +if resource is new and has added IP the action `run` can be inferred because previous state was +`None` and current is something new. If on the other hand previous state was some value `A` and +new state is some value `B` -- the orchestrator decides that the action to be run is `update`. And +if previous state is some `A` and new state is `None` the action will be `remove`. + +The 'appropriate order' taken by orchestrator can be just like the data flow graph initially. We +see possibility of optimizing the number of actions taken by orchestrator so that moving Keystone +service to another node can be simplified from 4 actions taken to 3 actions. + +After resource action is finished the new state is saved to the commit log and data is updated in +the DB. + +We want to support rollbacks via commit log. Rollback is done by replaying the commit log backwards. + +In case of separate commit logs per resource we think rollback could be done like this: some resource +`K` is rolled back by one commit log, the diff action is the same as reversed diff action of the +commit we are rolling back. We can update other resources with this new data by analyzing the connections. +So in other words -- we change the data in one resource according to what is in the commit to be rolled +back and then we trigger changes in other connected resources. Then we run orchestrator actions like +described above. + +From analysis of resource removal we think that we need to save connection data in each commit -- +otherwise when we rollback that resource removal we wouldn't know how to restore its connections +to other resources. From bbb856e4ad131cc48a166aede7daff1805a3f0e4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Fri, 22 May 2015 13:52:15 +0200 Subject: [PATCH 5/6] Docs: commit-log update --- docs/commit-log.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/commit-log.md b/docs/commit-log.md index bfdaf32c..5a656372 100644 --- a/docs/commit-log.md +++ b/docs/commit-log.md @@ -28,7 +28,10 @@ if previous state is some `A` and new state is `None` the action will be `remove The 'appropriate order' taken by orchestrator can be just like the data flow graph initially. We see possibility of optimizing the number of actions taken by orchestrator so that moving Keystone -service to another node can be simplified from 4 actions taken to 3 actions. +service to another node can be simplified from 4 actions (update HAProxy without removed Keystone, +install Keystone on new node, update HAProxy with new Keystone, remove Keystone from old node) +taken to 3 actions (add Keystone to new node, update HAProxy removing old Keystone and adding +new one, remove Keystone from old node). After resource action is finished the new state is saved to the commit log and data is updated in the DB. @@ -42,6 +45,14 @@ So in other words -- we change the data in one resource according to what is in back and then we trigger changes in other connected resources. Then we run orchestrator actions like described above. +In case of single commit log for all resources -- is it sufficient to just rollback a commit? Or +do we need to trigger changes in connected resources too? In global commit log we have ordering +of commits like they were run by orchestrator. + From analysis of resource removal we think that we need to save connection data in each commit -- otherwise when we rollback that resource removal we wouldn't know how to restore its connections to other resources. + +Single commits after every action finished on a resource causes many commits per one user 'apply' +action. In order to allow user to revert the whole action and not just single commits we have some +idea of 'tagging' group of commits by some action id. From 77564eb01ddd81460feb391a0589b63eb006cef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Mon, 25 May 2015 13:30:24 +0000 Subject: [PATCH 6/6] Fix example --- example.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/example.py b/example.py index 3ac24939..212a4aa2 100644 --- a/example.py +++ b/example.py @@ -1,5 +1,6 @@ import shutil import os +import time from solar.core import resource from solar.core import signals @@ -13,17 +14,16 @@ os.mkdir('rs') node1 = resource.create('node1', 'x/resources/ro_node/', 'rs/', {'ip':'10.0.0.3', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) node2 = resource.create('node2', 'x/resources/ro_node/', 'rs/', {'ip':'10.0.0.4', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) -node3 = resource.create('node3', 'x/resources/ro_node/', 'rs/', {'ip':'10.0.0.5', 'ssh_key' : '/vagrant/tmp/keys/ssh_private', 'ssh_user':'vagrant'}) mariadb_service1 = resource.create('mariadb_service1', 'x/resources/mariadb_service', 'rs/', {'image':'mariadb', 'root_password' : 'mariadb', 'port' : '3306', 'ip': '', 'ssh_user': '', 'ssh_key': ''}) keystone_db = resource.create('keystone_db', 'x/resources/mariadb_db/', 'rs/', {'db_name':'keystone_db', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''}) keystone_db_user = resource.create('keystone_db_user', 'x/resources/mariadb_user/', 'rs/', {'new_user_name' : 'keystone', 'new_user_password' : 'keystone', 'db_name':'', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''}) keystone_config1 = resource.create('keystone_config1', 'x/resources/keystone_config/', 'rs/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''}) -keystone_service1 = resource.create('keystone_service1', 'x/resources/keystone_service/', 'rs/', {'port':'5000', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) +keystone_service1 = resource.create('keystone_service1', 'x/resources/keystone_service/', 'rs/', {'port':'5001', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) keystone_config2 = resource.create('keystone_config2', 'x/resources/keystone_config/', 'rs/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''}) -keystone_service2 = resource.create('keystone_service2', 'x/resources/keystone_service/', 'rs/', {'port':'5000', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) +keystone_service2 = resource.create('keystone_service2', 'x/resources/keystone_service/', 'rs/', {'port':'5002', 'admin_port':'35357', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':'', 'config_dir':''}) haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'x/resources/haproxy_config/', 'rs/', {'name':'keystone_config', 'listen_port':'5000', 'servers':[], 'ports':[]}) @@ -62,11 +62,12 @@ signals.connect(node2, keystone_service2) signals.connect(keystone_config2, keystone_service2, {'config_dir': 'config_dir'}) signals.connect(keystone_service1, haproxy_keystone_config, {'ip':'servers', 'port':'ports'}) +signals.connect(keystone_service2, haproxy_keystone_config, {'ip':'servers', 'port':'ports'}) -signals.connect(node1, haproxy_config) +signals.connect(node2, haproxy_config) signals.connect(haproxy_keystone_config, haproxy_config, {'listen_port': 'listen_ports', 'name':'configs_names', 'ports' : 'configs_ports', 'servers':'configs'}) -signals.connect(node1, haproxy_service) +signals.connect(node2, haproxy_service) signals.connect(haproxy_config, haproxy_service, {'listen_ports':'ports', 'config_dir':'host_binds'}) @@ -74,19 +75,21 @@ signals.connect(haproxy_config, haproxy_service, {'listen_ports':'ports', 'confi from solar.core import actions actions.resource_action(mariadb_service1, 'run') +time.sleep(10) actions.resource_action(keystone_db, 'run') actions.resource_action(keystone_db_user, 'run') actions.resource_action(keystone_config1, 'run') actions.resource_action(keystone_service1, 'run') +actions.resource_action(keystone_service2, 'run') actions.resource_action(haproxy_config, 'run') actions.resource_action(haproxy_service, 'run') #remove -actions.resource_action(haproxy_service, 'remove') -actions.resource_action(haproxy_config, 'remove') -actions.resource_action(keystone_service1, 'remove') -actions.resource_action(keystone_config1, 'remove') -actions.resource_action(keystone_db_user, 'remove') -actions.resource_action(keystone_db, 'remove') -actions.resource_action(mariadb_service1, 'remove') +#actions.resource_action(haproxy_service, 'remove') +#actions.resource_action(haproxy_config, 'remove') +#actions.resource_action(keystone_service1, 'remove') +#actions.resource_action(keystone_config1, 'remove') +#actions.resource_action(keystone_db_user, 'remove') +#actions.resource_action(keystone_db, 'remove') +#actions.resource_action(mariadb_service1, 'remove')