From 3f0547d6eb3dc3d52feae101b3c81d1b1e63aff0 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 14 May 2015 11:07:43 +0200 Subject: [PATCH 1/7] Remove data action --- solar/solar/cli.py | 50 ---------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 9f7b212d..bc05827a 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -79,9 +79,6 @@ class Cmd(object): group.add_argument('-t', '--tags', nargs='+', default=['env/test_env']) group.add_argument('-i', '--id', default=utils.generate_uuid()) - parser = self.subparser.add_parser('data') - parser.set_defaults(func=getattr(self, 'data')) - def profile(self, args): if args.create: params = {'tags': args.tags, 'id': args.id} @@ -103,53 +100,6 @@ class Cmd(object): def discover(self, args): Discovery({'id': 'discovery'}).discover() - def data(self, args): - - resources = [ - {'id': 'mariadb', - 'tags': ['service/mariadb', 'entrypoint/mariadb'], - 'input': { - 'ip_addr': '{{ this.node.ip }}' }}, - - {'id': 'keystone', - 'tags': ['service/keystone'], - 'input': { - 'name': 'keystone-test', - 'admin_port': '35357', - 'public_port': '5000', - 'db_addr': {'first_with_tags': ["entrypoint/mariadb"], 'item': '{{ item.node.ip }}'}}}, - - {'id': 'haproxy', - 'tags': ['service/haproxy'], - 'input': { - 'services': [ - {'service_name': 'keystone-admin', - 'bind': '*:8080', - 'backends': { - 'with_tags': ["service/keystone"], - 'item': {'name': '{{ item.name }}', 'addr': '{{ item.node.ip }}:{{ item.admin_port }}'}}}, - - {'service_name': 'keystone-pub', - 'bind': '*:8081', - 'backends': { - 'with_tags': ["service/keystone"], - 'item': {'name': '{{ item.name }}', 'addr': '{{ item.node.ip }}:{{ item.public_port }}'}}}]}} - ] - - nodes = [ - {'id': 'n-1', - 'ip': '10.0.0.2', - 'tags': ['node/1', 'service/keystone']}, - - {'id': 'n-2', - 'ip': '10.0.0.3', - 'tags': ['node/2', 'service/mariadb', 'service/haproxy', 'service/keystone']}] - - dg = data.DataGraph(resources, nodes) - - pprint.pprint(dg.resolve()) - - def main(): api = Cmd() From 27f85f4d3101ef9b50e089555d01208c64a995c5 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 14 May 2015 14:59:17 +0200 Subject: [PATCH 2/7] Implemented tags filtration language --- solar/requirements.txt | 2 +- solar/solar/core/tags_set_parser.py | 117 ++++++++++++++++++++++++++++ solar/solar/errors.py | 3 + 3 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 solar/solar/core/tags_set_parser.py diff --git a/solar/requirements.txt b/solar/requirements.txt index c8ba07ba..55598e7e 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -3,4 +3,4 @@ pyyaml jinja2 networkx mock - +ply diff --git a/solar/solar/core/tags_set_parser.py b/solar/solar/core/tags_set_parser.py new file mode 100644 index 00000000..b8cb25c9 --- /dev/null +++ b/solar/solar/core/tags_set_parser.py @@ -0,0 +1,117 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ply import lex +from ply import yacc + +from solar import errors + + +tokens = ( + "STRING", + "AND", + "OR", + "LPAREN", + "RPAREN") + +t_STRING = r'[A-Za-z0-9-_/\\]+' +t_AND = '&&' +t_OR = r'\|\|' +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_ignore = ' \t\r\n' + + +class SubexpressionWrapper(object): + + def __init__(self, subexpression): + self.subexpression = subexpression + + def evaluate(self): + self.value = self.subexpression() + return self.value + + def __call__(self): + self.evaluate() + return self.value + + +class ScalarWrapper(object): + def __init__(self, value): + global expression + self.value = (set([value]) <= set(expression.tags)) + + def evaluate(self): + return self.value + + def __call__(self): + return self.value + + +def p_expression_logical_op(p): + """expression : expression AND expression + | expression OR expression + """ + result, arg1, op, arg2 = p + if op == '&&': + result = lambda: arg1() and arg2() + elif op == '||': + result = lambda: arg1() or arg2() + + p[0] = SubexpressionWrapper(result) + + +def p_expression_string(p): + """expression : STRING + """ + p[0] = ScalarWrapper(p[1]) + + +def p_expression_group(p): + """expression : LPAREN expression RPAREN + """ + p[0] = p[2] + + +def t_error(t): + errors.LexError("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + + +def p_error(p): + raise errors.ParseError("Syntax error at '{0}'".format(getattr(p, 'value', ''))) + + +class Expression(object): + def __init__(self, expression_text, tags): + self.expression_text = expression_text + self.tags = tags + self.compiled_expression = parse(self) + + def evaluate(self): + return self.compiled_expression() + + +lexer = lex.lex() +parser = yacc.yacc(debug=False, write_tables=False) +expression = None + + +def parse(expr): + global parser + global expression + # TODO it's very very bad to have global variable here + # we should figure a way to pass it into ScalarWrapper + expression = expr + return parser.parse(expr.expression_text) diff --git a/solar/solar/errors.py b/solar/solar/errors.py index 3f38904c..ec289e36 100644 --- a/solar/solar/errors.py +++ b/solar/solar/errors.py @@ -9,3 +9,6 @@ class CannotFindID(SolarError): class CannotFindExtension(SolarError): pass + +class ParseError(SolarError): + pass From 85c9fcafa7dee6ced593c7c8ab4dd7c9702fc989 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 14 May 2015 15:14:53 +0200 Subject: [PATCH 3/7] Added assign command and implemented resources and nodes retrival --- config.yml | 1 + solar/solar/cli.py | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/config.yml b/config.yml index 400e4590..5ced636f 100644 --- a/config.yml +++ b/config.yml @@ -9,3 +9,4 @@ file-system-db: storage-path: /vagrant/tmp/storage template-dir: /vagrant/templates +resources-files-mask: /vagrant/examples/resources/*.yml diff --git a/solar/solar/cli.py b/solar/solar/cli.py index bc05827a..f5f1ee24 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + """Solar CLI api On create "golden" resource should be moved to special place @@ -27,6 +28,7 @@ import yaml from solar import extensions from solar import utils from solar.core import data +from solar.core.tags_set_parser import Expression from solar.interfaces.db import get_db # NOTE: these are extensions, they shouldn't be imported here @@ -79,12 +81,17 @@ class Cmd(object): group.add_argument('-t', '--tags', nargs='+', default=['env/test_env']) group.add_argument('-i', '--id', default=utils.generate_uuid()) + # Assign + parser = self.subparser.add_parser('assign') + parser.set_defaults(func=getattr(self, 'assign')) + parser.add_argument('-n', '--nodes') + parser.add_argument('-r', '--resources') + def profile(self, args): if args.create: params = {'tags': args.tags, 'id': args.id} profile_template_path = os.path.join( - utils.read_config()['template-dir'], 'profile.yml' - ) + utils.read_config()['template-dir'], 'profile.yml') data = yaml.load(utils.render_template(profile_template_path, params)) self.db.store('profiles', data) else: @@ -100,6 +107,18 @@ class Cmd(object): def discover(self, args): Discovery({'id': 'discovery'}).discover() + def assign(self, args): + nodes = filter( + lambda n: Expression(args.nodes, n['tags']).evaluate(), + self.db.get_list('nodes')) + + resources = filter( + lambda r: Expression(args.resources, r['tags']).evaluate(), + self._get_resources_list()) + + def _get_resources_list(self): + return utils.load_by_mask(utils.read_config()['resources-files-mask']) + def main(): api = Cmd() From da3724cd3f436c8eb1cb9a6ebb15b02d87c8fcb0 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 14 May 2015 16:02:40 +0200 Subject: [PATCH 4/7] Implemented resources to nodes assignment --- config.yml | 3 ++- solar/solar/cli.py | 21 ++++++++++++++++++--- solar/solar/core/resource.py | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/config.yml b/config.yml index 5ced636f..0889c6bb 100644 --- a/config.yml +++ b/config.yml @@ -9,4 +9,5 @@ file-system-db: storage-path: /vagrant/tmp/storage template-dir: /vagrant/templates -resources-files-mask: /vagrant/examples/resources/*.yml +resources-files-mask: /vagrant/x/resources/*/*.yaml +resource-instances-path: /vagrant/tmp/resource-instances diff --git a/solar/solar/cli.py b/solar/solar/cli.py index f5f1ee24..ea119d6a 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -28,6 +28,7 @@ import yaml from solar import extensions from solar import utils from solar.core import data +from solar.core.resource import assign_resources_to_nodes from solar.core.tags_set_parser import Expression from solar.interfaces.db import get_db @@ -109,15 +110,29 @@ class Cmd(object): def assign(self, args): nodes = filter( - lambda n: Expression(args.nodes, n['tags']).evaluate(), + lambda n: Expression(args.nodes, n.get('tags', [])).evaluate(), self.db.get_list('nodes')) resources = filter( - lambda r: Expression(args.resources, r['tags']).evaluate(), + lambda r: Expression(args.resources, r.get('tags', [])).evaluate(), self._get_resources_list()) + resource_instances_path = utils.read_config()['resource-instances-path'] + utils.create_dir(resource_instances_path) + assign_resources_to_nodes( + resources, + nodes, + resource_instances_path) + def _get_resources_list(self): - return utils.load_by_mask(utils.read_config()['resources-files-mask']) + result = [] + for path in utils.find_by_mask(utils.read_config()['resources-files-mask']): + resource = utils.yaml_load(path) + resource['path'] = path + resource['dir_path'] = os.path.dirname(path) + result.append(resource) + + return result def main(): diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index 93977bd5..10431af1 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -4,6 +4,8 @@ import json import os import shutil +from copy import deepcopy + import yaml from solar.core import actions @@ -174,3 +176,19 @@ def load_all(dest_path): signals.Connections.reconnect_all() return ret + + +def assign_resources_to_nodes(resources, nodes, dst_dir): + for node in nodes: + for resource in resources: + merged = deepcopy(resource) + # Node specific setting should override resource's + merged.update(deepcopy(node)) + # Tags for specific resource is set of tags from node and from resource + merged['tags'] = list(set(node.get('tags', [])) + set(resource.get('tags', []))) + + create( + format('{0}-{1}'.format(node['id'], resource['id'])), + resource['dir_path'], + dst_dir, + merged) From ed634baf2f0a7b5cbbed9a70b874e6db29ec2495 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Thu, 14 May 2015 16:09:40 +0200 Subject: [PATCH 5/7] Update readme --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 47ff4431..e8dbf4ea 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,9 @@ vim tmp/storage/nodes-id.yaml ``` * add `env/test_env` in tags list -* in order to assign resouce to the node use the same the same - method, i.e. add in tags list for node your service e.g. - `service/docker`, `service/mariadb` -* perform deployment +* assign resources to nodes ``` -solar configure --profile prf1 -pa run +# TODO Does not work without default values in golden templates +solar assign -n "env/test_env && node/1" -r resource/mariadb ``` From d5862e83feabee7c7a30c30ab7f8c977749992bf Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Fri, 15 May 2015 12:57:07 +0200 Subject: [PATCH 6/7] Fix tags concatination for resources creation --- solar/solar/core/resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index 10431af1..fbfe007e 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -185,7 +185,8 @@ def assign_resources_to_nodes(resources, nodes, dst_dir): # Node specific setting should override resource's merged.update(deepcopy(node)) # Tags for specific resource is set of tags from node and from resource - merged['tags'] = list(set(node.get('tags', [])) + set(resource.get('tags', []))) + merged['tags'] = list(set(node.get('tags', [])) | + set(resource.get('tags', []))) create( format('{0}-{1}'.format(node['id'], resource['id'])), From 3b9c54622e4b56e7154ccfa13d526122d9c90240 Mon Sep 17 00:00:00 2001 From: Evgeniy L Date: Wed, 20 May 2015 12:19:54 +0200 Subject: [PATCH 7/7] Replaced && with "&" and "," operators and replaced "||" with "|" --- solar/solar/core/tags_set_parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solar/solar/core/tags_set_parser.py b/solar/solar/core/tags_set_parser.py index b8cb25c9..0a48b02a 100644 --- a/solar/solar/core/tags_set_parser.py +++ b/solar/solar/core/tags_set_parser.py @@ -26,8 +26,8 @@ tokens = ( "RPAREN") t_STRING = r'[A-Za-z0-9-_/\\]+' -t_AND = '&&' -t_OR = r'\|\|' +t_AND = '&|,' +t_OR = r'\|' t_LPAREN = r'\(' t_RPAREN = r'\)' t_ignore = ' \t\r\n' @@ -64,9 +64,9 @@ def p_expression_logical_op(p): | expression OR expression """ result, arg1, op, arg2 = p - if op == '&&': + if op == '&' or op == ',': result = lambda: arg1() and arg2() - elif op == '||': + elif op == '|': result = lambda: arg1() or arg2() p[0] = SubexpressionWrapper(result)