Merge pull request #52 from xarses/move-x-into-solar-core

Initial implementation of assign method

README.md

@@ -39,11 +39,9 @@ vim tmp/storage/nodes-id.yaml
 ```
 
 * add `env/test_env` in tags list
-* in order to assign resouce to the node use the same the same
+* assign resources to nodes
-  method, i.e. add in tags list for node your service e.g.
-  `service/docker`, `service/mariadb`
-* perform deployment
 
 ```
-solar configure --profile prf1 -pa run
+# TODO Does not work without default values in golden templates
+solar assign -n "env/test_env && node/1" -r resource/mariadb
 ```

config.yml

@@ -9,3 +9,5 @@ file-system-db:
   storage-path: /vagrant/tmp/storage
 
 template-dir: /vagrant/templates
+resources-files-mask: /vagrant/x/resources/*/*.yaml
+resource-instances-path: /vagrant/tmp/resource-instances

solar/requirements.txt

@@ -3,4 +3,4 @@ pyyaml
 jinja2
 networkx
 mock
-
+ply

solar/solar/cli.py

@@ -11,6 +11,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 """Solar CLI api
 
 On create "golden" resource should be moved to special place
@@ -27,6 +28,8 @@ import yaml
 from solar import extensions
 from solar import utils
 from solar.core import data
+from solar.core.resource import assign_resources_to_nodes
+from solar.core.tags_set_parser import Expression
 from solar.interfaces.db import get_db
 
 # NOTE: these are extensions, they shouldn't be imported here
@@ -79,15 +82,17 @@ class Cmd(object):
         group.add_argument('-t', '--tags', nargs='+', default=['env/test_env'])
         group.add_argument('-i', '--id', default=utils.generate_uuid())
 
-        parser = self.subparser.add_parser('data')
+        # Assign
-        parser.set_defaults(func=getattr(self, 'data'))
+        parser = self.subparser.add_parser('assign')
+        parser.set_defaults(func=getattr(self, 'assign'))
+        parser.add_argument('-n', '--nodes')
+        parser.add_argument('-r', '--resources')
 
     def profile(self, args):
         if args.create:
             params = {'tags': args.tags, 'id': args.id}
             profile_template_path = os.path.join(
-                utils.read_config()['template-dir'], 'profile.yml'
+                utils.read_config()['template-dir'], 'profile.yml')
-            )
             data = yaml.load(utils.render_template(profile_template_path, params))
             self.db.store('profiles', data)
         else:
@@ -103,52 +108,31 @@ class Cmd(object):
     def discover(self, args):
         Discovery({'id': 'discovery'}).discover()
 
-    def data(self, args):
+    def assign(self, args):
+        nodes = filter(
+            lambda n: Expression(args.nodes, n.get('tags', [])).evaluate(),
+            self.db.get_list('nodes'))
 
-        resources = [
+        resources = filter(
-             {'id': 'mariadb',
+            lambda r: Expression(args.resources, r.get('tags', [])).evaluate(),
-              'tags': ['service/mariadb', 'entrypoint/mariadb'],
+            self._get_resources_list())
-              'input': {
-                  'ip_addr': '{{ this.node.ip }}' }},
 
-            {'id': 'keystone',
+        resource_instances_path = utils.read_config()['resource-instances-path']
-             'tags': ['service/keystone'],
+        utils.create_dir(resource_instances_path)
-             'input': {
+        assign_resources_to_nodes(
-                 'name': 'keystone-test',
+            resources,
-                 'admin_port': '35357',
+            nodes,
-                 'public_port': '5000',
+            resource_instances_path)
-                 'db_addr': {'first_with_tags': ["entrypoint/mariadb"], 'item': '{{ item.node.ip }}'}}},
 
-             {'id': 'haproxy',
+    def _get_resources_list(self):
-              'tags': ['service/haproxy'],
+        result = []
-              'input': {
+        for path in utils.find_by_mask(utils.read_config()['resources-files-mask']):
-                  'services': [
+            resource = utils.yaml_load(path)
-                      {'service_name': 'keystone-admin',
+            resource['path'] = path
-                       'bind': '*:8080',
+            resource['dir_path'] = os.path.dirname(path)
-                       'backends': {
+            result.append(resource)
-                           'with_tags': ["service/keystone"],
-                           'item': {'name': '{{ item.name }}', 'addr': '{{ item.node.ip }}:{{ item.admin_port }}'}}},
-
-                      {'service_name': 'keystone-pub',
-                       'bind': '*:8081',
-                       'backends': {
-                           'with_tags': ["service/keystone"],
-                           'item': {'name': '{{ item.name }}', 'addr': '{{ item.node.ip }}:{{ item.public_port }}'}}}]}}
-        ]
-
-        nodes = [
-            {'id': 'n-1',
-             'ip': '10.0.0.2',
-             'tags': ['node/1', 'service/keystone']},
-
-            {'id': 'n-2',
-             'ip': '10.0.0.3',
-             'tags': ['node/2', 'service/mariadb', 'service/haproxy', 'service/keystone']}]
-
-        dg = data.DataGraph(resources, nodes)
-
-        pprint.pprint(dg.resolve())
 
+        return result
 
 
 def main():

solar/solar/core/resource.py

@@ -4,6 +4,8 @@ import json
 import os
 import shutil
 
+from copy import deepcopy
+
 import yaml
 
 from solar.core import actions
@@ -174,3 +176,20 @@ def load_all(dest_path):
     signals.Connections.reconnect_all()
 
     return ret
+
+
+def assign_resources_to_nodes(resources, nodes, dst_dir):
+    for node in nodes:
+        for resource in resources:
+            merged = deepcopy(resource)
+            # Node specific setting should override resource's
+            merged.update(deepcopy(node))
+            # Tags for specific resource is set of tags from node and from resource
+            merged['tags'] = list(set(node.get('tags', [])) |
+                                  set(resource.get('tags', [])))
+
+            create(
+                format('{0}-{1}'.format(node['id'], resource['id'])),
+                resource['dir_path'],
+                dst_dir,
+                merged)

solar/solar/core/tags_set_parser.py

@@ -0,0 +1,117 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from ply import lex
+from ply import yacc
+
+from solar import errors
+
+
+tokens = (
+    "STRING",
+    "AND",
+    "OR",
+    "LPAREN",
+    "RPAREN")
+
+t_STRING = r'[A-Za-z0-9-_/\\]+'
+t_AND  = '&|,'
+t_OR  = r'\|'
+t_LPAREN  = r'\('
+t_RPAREN  = r'\)'
+t_ignore = ' \t\r\n'
+
+
+class SubexpressionWrapper(object):
+
+    def __init__(self, subexpression):
+        self.subexpression = subexpression
+
+    def evaluate(self):
+        self.value = self.subexpression()
+        return self.value
+
+    def __call__(self):
+        self.evaluate()
+        return self.value
+
+
+class ScalarWrapper(object):
+    def __init__(self, value):
+        global expression
+        self.value = (set([value]) <= set(expression.tags))
+
+    def evaluate(self):
+        return self.value
+
+    def __call__(self):
+        return self.value
+
+
+def p_expression_logical_op(p):
+    """expression : expression AND expression
+                  | expression OR expression
+    """
+    result, arg1, op, arg2 = p
+    if op == '&' or op == ',':
+        result = lambda: arg1() and arg2()
+    elif op == '|':
+        result = lambda: arg1() or arg2()
+
+    p[0] = SubexpressionWrapper(result)
+
+
+def p_expression_string(p):
+    """expression : STRING
+    """
+    p[0] = ScalarWrapper(p[1])
+
+
+def p_expression_group(p):
+    """expression : LPAREN expression RPAREN
+    """
+    p[0] = p[2]
+
+
+def t_error(t):
+    errors.LexError("Illegal character '%s'" % t.value[0])
+    t.lexer.skip(1)
+
+
+def p_error(p):
+    raise errors.ParseError("Syntax error at '{0}'".format(getattr(p, 'value', '')))
+
+
+class Expression(object):
+    def __init__(self, expression_text, tags):
+        self.expression_text = expression_text
+        self.tags = tags
+        self.compiled_expression = parse(self)
+
+    def evaluate(self):
+        return self.compiled_expression()
+
+
+lexer = lex.lex()
+parser = yacc.yacc(debug=False, write_tables=False)
+expression = None
+
+
+def parse(expr):
+    global parser
+    global expression
+    # TODO it's very very bad to have global variable here
+    # we should figure a way to pass it into ScalarWrapper
+    expression = expr
+    return parser.parse(expr.expression_text)

solar/solar/errors.py

@@ -9,3 +9,6 @@ class CannotFindID(SolarError):
 class CannotFindExtension(SolarError):
     pass
 
+
+class ParseError(SolarError):
+    pass