Merge pull request #208 from loles/vr_tags

Vr tags

examples/riak/haproxy_riak_config.yaml

@@ -0,0 +1,50 @@
+id: haproxy_riak_config
+
+resources:
+  - id: haproxy_riak_config_http
+    from: resources/haproxy_service_config
+    tags: ['service=riak', 'protocol=http']
+    values:
+      listen_port: {{http_listen_port}}
+      protocol: 'http'
+      name: 'riak_haproxy_http'
+      backends:server:
+        {% for riak in riaks %}
+        - {{riak}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for riak in riaks %}
+        - {{riak}}::riak_port_http
+        {% endfor %}
+
+  - id: haproxy_riak_config_pb
+    from: resources/haproxy_service_config
+    tags: ['service=riak', 'protocol=tcp']
+    values:
+      listen_port: {{pb_listen_port}}
+      protocol: 'tcp'
+      name: 'riak_haproxy_pb'
+      backends:server:
+        {% for riak in riaks %}
+        - {{riak}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for riak in riaks %}
+        - {{riak}}::riak_port_pb
+        {% endfor %}
+
+updates:
+  - with_tags: ['resource=haproxy_config']
+    values:
+      config:protocol:
+        - haproxy_riak_config_http::protocol
+        - haproxy_riak_config_pb::protocol
+      config:listen_port:
+        - haproxy_riak_config_http::listen_port
+        - haproxy_riak_config_pb::listen_port
+      config:name:
+        - haproxy_riak_config_http::name
+        - haproxy_riak_config_pb::name
+      config:backends:
+        - haproxy_riak_config_http::backends
+        - haproxy_riak_config_pb::backends

examples/riak/riak_cluster.yaml

@@ -1,139 +1,51 @@
-id: riak_node
+id: riak_cluster
 
 resources:
-  - id: riak_service0
-    from: resources/riak_node
-    location: {{nodes[0]}}
+  - id: riak_service1
+    from: examples/riak/riak_service.yaml
     values:
-      riak_self_name: 'riak0'
-      riak_hostname: 'riak_server0.solar'
-      riak_name: 'riak0@riak_server0.solar'
-      ip: '{{nodes[0]}}::ip'
+      node: {{nodes[0]}}
+      index: 1 
+      join_to: ''
 
-{% for i in range(1, idx|int) %}
-  - id: riak_service{{i}}
-    from: resources/riak_node
-    location: {{nodes[i]}}
+  - id: riak_service2
+    from: examples/riak/riak_service.yaml
     values:
-      riak_self_name: 'riak{{i}}'
-      riak_hostname: 'riak_server{{i}}.solar'
-      riak_name: 'riak{{i}}@riak_server{{i}}.solar'
-      join_to: 'riak_service0::riak_name'
-      ip: '{{nodes[i]}}::ip'
-{% endfor %}
+      node: {{nodes[1]}}
+      index: 2 
+      join_to: riak_service1
 
-{% for i in range(idx|int) %}
-  - id: hosts_file{{i}}
-    from: resources/hosts_file
-    location: {{nodes[i]}}
+  - id: riak_service3
+    from: examples/riak/riak_service.yaml
     values:
-      hosts:name: 
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::riak_hostname::NO_EVENTS
-        {% endfor %}
-      hosts:ip:
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::ip::NO_EVENTS
-        {% endfor %}
-{% endfor %}
+      node: {{nodes[2]}}
+      index: 3 
+      join_to: riak_service1
 
-  - id: haproxy_riak_config_http
-    from: resources/haproxy_service_config
+  - id: haproxy_riak_config
+    from: examples/riak/haproxy_riak_config.yaml
     values:
-      listen_port: 8098
-      protocol: 'http'
-      name: 'riak_haproxy_http'
-      backends:server:
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::riak_hostname
-        {% endfor %}
-      backends:port:
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::riak_port_http
-        {% endfor %}
+      http_listen_port: 8098
+      pb_listen_port: 8087
+      riaks: ['riak_service1', 'riak_service2', 'riak_service3']
 
-  - id: haproxy_riak_config_pb
-    from: resources/haproxy_service_config
+  - id: haproxy1
+    from: templates/haproxy.yaml
     values:
-      listen_port: 8087
-      protocol: 'tcp'
-      name: 'riak_haproxy_pb'
-      backends:server:
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::riak_hostname
-        {% endfor %}
-      backends:port:
-        {% for j in range(idx|int) %}
-        - riak_service{{j}}::riak_port_pb
-        {% endfor %}
+      node: {{nodes[0]}}
+      service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http']
+      index: 1
 
-{% for i in range(idx|int) %}
-  - id: haproxy_config{{i}}
-    from: resources/haproxy_config
-    location: {{nodes[i]}}
+  - id: haproxy2
+    from: templates/haproxy.yaml
     values:
-      config:protocol:
-        - haproxy_riak_config_http::protocol
-        - haproxy_riak_config_pb::protocol
-      config:listen_port:
-        - haproxy_riak_config_http::listen_port
-        - haproxy_riak_config_pb::listen_port
-      config:name:
-        - haproxy_riak_config_http::name
-        - haproxy_riak_config_pb::name
-      config:backends:
-        - haproxy_riak_config_http::backends
-        - haproxy_riak_config_pb::backends
-      ip: '{{nodes[i]}}::ip'
-{% endfor %}
+      node: {{nodes[1]}}
+      service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http']
+      index: 2
 
-{% for i in range(idx|int) %}
-  - id: haproxy_service{{i}}
-    location: {{nodes[i]}}
-    from: resources/haproxy_service
+  - id: haproxy3
+    from: templates/haproxy.yaml
     values:
-      ip: '{{nodes[i]}}::ip'
-{% endfor %}
-
-
-events:
-{% for i in range(idx|int) %}
-  - type: depends_on
-    parent_action: 'hosts_file{{i}}.run'
-    state: 'success'
-    depend_action: 'riak_service{{i}}.run'
-{% endfor %}
-
-{% for i in range(1, idx|int) %}
-  - type: react_on
-    parent_action: 'riak_service{{i}}.run'
-    state: 'success'
-    depend_action: 'riak_service{{i}}.join'
-
-  - type: react_on
-    parent_action: 'riak_service{{i}}.leave'
-    state: 'success'
-    depend_action: 'riak_service{{i}}.join'
-
-  - type: react_on
-    parent_action: 'riak_service{{i}}.join'
-    state: 'success'
-    depend_action: 'riak_service0.commit'
-{% endfor %}
-
-{% for i in range(1, idx|int) %}
-  - type: depends_on
-    parent_action: 'haproxy_service{{i}}.run'
-    state: 'success'
-    depend_action: 'haproxy_config{{i}}.run'
-
-  - type: react_on
-    parent_action: 'haproxy_config{{i}}.run'
-    state: 'success'
-    depend_action: 'haproxy_service{{i}}.apply_config'
-
-  - type: react_on
-    parent_action: 'haproxy_config{{i}}.update'
-    state: 'success'
-    depend_action: 'haproxy_service{{i}}.apply_config'
-{% endfor %}
+      node: {{nodes[2]}}
+      service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http']
+      index: 3

examples/riak/riak_service.yaml

@@ -0,0 +1,61 @@
+id: riak_service
+
+resources:
+  - id: riak_service{{index}}
+    from: resources/riak_node
+    location: {{node}}
+    values:
+      riak_self_name: riak{{index}}
+      riak_hostname: riak_server{{index}}.solar
+      riak_name: riak{{index}}@riak_server{{index}}.solar
+    {% if join_to %}
+      join_to: {{join_to}}::riak_name
+    {% endif %}
+      ip: {{node}}::ip
+
+updates:
+  - with_tags: ['resource=hosts_file']
+    values:
+      hosts:name: 
+        - riak_service{{index}}::riak_hostname::NO_EVENTS
+      hosts:ip:
+        - riak_service{{index}}::ip::NO_EVENTS
+
+  - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=http'] 
+    values:
+      backends:server:
+        - riak_service{{index}}::riak_hostname
+      backends:port:
+        - riak_service{{index}}::riak_port_http
+
+  - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=tcp'] 
+    values:
+      backends:server:
+        - riak_service{{index}}::riak_hostname
+      backends:port:
+        - riak_service{{index}}::riak_port_pb
+
+events:
+  - type: depends_on
+    parent: 
+      with_tags: ['resource=hosts_file', 'location={{node}}']
+      action: run
+    state: success
+    depend_action: riak_service{{index}}.run
+
+{% if join_to %}
+  - type: react_on
+    parent_action: riak_service{{index}}.run
+    state: success
+    depend_action: riak_service{{index}}.join
+
+  - type: react_on
+    parent_action: riak_service{{index}}.leave
+    state: success
+    depend_action: riak_service{{index}}.join
+
+  - type: react_on
+    parent_action: riak_service{{index}}.join
+    state: success
+    depend_action: {{join_to}}.commit
+{% endif %}

resources/haproxy_config/meta.yaml

@@ -18,4 +18,4 @@ input:
     #   schema: str!
     #   value:
 
-tags: [resources/haproxy, resource/haproxy_config]
+tags: [resources=haproxy]

resources/haproxy_service/meta.yaml

@@ -12,4 +12,4 @@ input:
     #   schema: str!
     #   value:
 
-tags: [resources/haproxy, resource/haproxy_service]
+tags: [resources=haproxy]

resources/haproxy_service_config/meta.yaml

@@ -1,4 +1,4 @@
-id: haproxy_general_config
+id: haproxy_service_config
 handler: none
 version: 1.0.0
 input:
@@ -20,4 +20,4 @@ input:
     #   schema: [str]
     #   value: []
 
-tags: [resources/haproxy, resource/haproxy_general_config]
+tags: [resources=haproxy]

resources/hosts_file/actions/run.yaml

@@ -5,7 +5,7 @@
     - name: Create hosts entries for {{val['name']}} => {{val['ip']}}
       lineinfile:
         dest: /etc/hosts
-        regexp: '.*{{val['name']}}$'
-        line: '{{val['ip']}} {{val['name']}}'
+        regexp: ".*{{val['name']}}$"
+        line: "{{val['ip']}} {{val['name']}}"
         state: present
     {% endfor %}

resources/hosts_file/actions/update.yaml

@@ -0,0 +1,11 @@
+- hosts: [{{host}}]
+  sudo: yes
+  tasks:
+    {% for val in hosts %}
+    - name: Create hosts entries for {{val['name']}} => {{val['ip']}}
+      lineinfile:
+        dest: /etc/hosts
+        regexp: ".*{{val['name']}}$"
+        line: "{{val['ip']}} {{val['name']}}"
+        state: present
+    {% endfor %}

resources/hosts_file/meta.yaml

@@ -5,3 +5,6 @@ version: 1.0.0
 input:
     hosts:
       schema: [{name: str!, ip: str!}]
+      value: []
+
+tags: [resource=hosts_file]

resources/ro_node/meta.yaml

@@ -1,4 +1,4 @@
-id: mariadb
+id: ro_node
 handler: none
 version: 1.0.0
 actions:
@@ -19,3 +19,5 @@ input:
       schema: str!
       value: $uuid
       reverse: True
+
+tags: [resources=node]

solar/solar/cli/resource.py

@@ -162,12 +162,13 @@ def show(name, tag, json, color):
 @click.argument('tags', nargs=-1)
 @click.option('--add/--delete', default=True)
 def tag(add, tags, resource_name):
-    click.echo('Tag {} with {} {}'.format(resource_name, tags, add))
     r = sresource.load(resource_name)
     if add:
         r.add_tags(*tags)
+        click.echo('Tag(s) {} added to {}'.format(tags, resource_name))
     else:
         r.remove_tags(*tags)
+        click.echo('Tag(s) {} removed from {}'.format(tags, resource_name))
 
 @resource.command()
 @click.argument('name')

solar/solar/core/resource/resource.py

@@ -59,6 +59,13 @@ class Resource(object):
             metadata = deepcopy(self._metadata)
 
         self.base_path = base_path
+
+        if tags is None:
+            tags = []
+        m_tags = metadata.get('tags', [])
+        tags.extend(m_tags)
+        tags.append('resource={}'.format(metadata['id']))
+
         self.virtual_resource = virtual_resource
 
         inputs = metadata.get('input', {})
@@ -74,7 +81,8 @@ class Resource(object):
             'handler': metadata.get('handler', ''),
             'puppet_module': metadata.get('puppet_module', ''),
             'version': metadata.get('version', ''),
-            'meta_inputs': inputs
+            'meta_inputs': inputs,
+            'tags': tags
 
         })
         self.db_obj.state = RESOURCE_STATE.created.name
@@ -89,7 +97,6 @@ class Resource(object):
         self.db_obj = resource_db
         self.name = resource_db.name
         self.base_path = resource_db.base_path
-        # TODO: tags
         self.virtual_resource = None
 
     def auto_extend_inputs(self, inputs):
@@ -255,8 +262,8 @@ def load(name):
 def load_all():
     return [Resource(r) for r in orm.DBResource.load_all()]
 
-
 def load_by_tags(tags):
+    tags = set(tags)
     return [Resource(r) for r in orm.DBResource.load_all()
             if tags.issubset(set(r.tags))]
 

solar/solar/core/resource/virtual_resource.py

@@ -21,13 +21,14 @@ from jinja2 import Template, Environment, meta
 
 from solar.core import provider
 from solar.core import signals
+from solar.core.log import log
 from solar.core.resource import load as load_resource
-from solar.core.resource import Resource
+from solar.core.resource import Resource, load_by_tags
 from solar.events.api import add_event
 from solar.events.controls import React, Dep
 
 
-def create(name, base_path, args=None, virtual_resource=None):
+def create(name, base_path, args=None, tags=None, virtual_resource=None):
     args = args or {}
     if isinstance(base_path, provider.BaseProvider):
         base_path = base_path.directory
@@ -40,18 +41,19 @@ def create(name, base_path, args=None, virtual_resource=None):
     if is_virtual(base_path):
         template = _compile_file(name, base_path, args)
         yaml_template = yaml.load(StringIO(template))
-        rs = create_virtual_resource(name, yaml_template)
+        rs = create_virtual_resource(name, yaml_template, tags)
     else:
         r = create_resource(name,
                             base_path,
                             args=args,
+                            tags=tags,
                             virtual_resource=virtual_resource)
         rs = [r]
 
     return rs
 
 
-def create_resource(name, base_path, args=None, virtual_resource=None):
+def create_resource(name, base_path, args=None, tags=None, virtual_resource=None):
     args = args or {}
     if isinstance(base_path, provider.BaseProvider):
         base_path = base_path.directory
@@ -59,17 +61,17 @@ def create_resource(name, base_path, args=None, virtual_resource=None):
     # List args init with empty list. Elements will be added later
     args = {key: (value if not isinstance(value, list) else []) for key, value in args.items()}
     r = Resource(
-        name, base_path, args=args, tags=[], virtual_resource=virtual_resource
+        name, base_path, args=args, tags=tags, virtual_resource=virtual_resource
     )
     return r
 
 
-def create_virtual_resource(vr_name, template):
+def create_virtual_resource(vr_name, template, tags=None):
     template_resources = template.get('resources', [])
     template_events = template.get('events', [])
     resources_to_update = template.get('updates', [])
 
-    created_resources = create_resources(template_resources)
+    created_resources = create_resources(template_resources, tags=tags)
     events = parse_events(template_events)
     for event in events:
         add_event(event)
@@ -111,27 +113,47 @@ def is_virtual(path):
     return os.path.isfile(path)
 
 
-def create_resources(resources):
+def create_resources(resources, tags=None):
     created_resources = []
     cwd = os.getcwd()
     for r in resources:
         resource_name = r['id']
-        args = r['values']
+        args = r.get('values', {})
         node = r.get('location', None)
         from_path = r.get('from', None)
+        tags = r.get('tags', [])
         base_path = os.path.join(cwd, from_path)
-        new_resources = create(resource_name, base_path)
+        new_resources = create(resource_name, base_path, args=args, tags=tags)
         created_resources += new_resources
         if not is_virtual(base_path):
             if node:
                 node = load_resource(node)
-                r = load_resource(resource_name)
+                r = new_resources[0]
                 signals.connect(node, r, {})
+                r.add_tags('location={}'.format(node.name))
             update_inputs(resource_name, args)
     return created_resources
 
 
-def update_resources(resources):
+def extend_resources(template_resources):
+    resources = []
+    for r in template_resources:
+        if r.get('id'):
+            resources.append(r)
+        if r.get('with_tags'):
+            tags = r.get('with_tags')
+            filtered = load_by_tags(tags)
+            for f in filtered:
+                r = {'id': f.name,
+                     'values': r['values']}
+                resources.append(r)
+                log.debug('Resource {} for tags {} found'.format(r, tags))
+            if not filtered:
+                log.debug('Warrning: no resources with tags: {}'.format(tags))
+    return resources
+
+def update_resources(template_resources):
+    resources = extend_resources(template_resources)
     for r in resources:
         resource_name = r['id']
         args = r['values']
@@ -151,8 +173,28 @@ def update_inputs(child, args):
     child.update(assignments)
 
 
-def parse_events(events):
+def extend_events(template_events):
+    events = []
+    for e in template_events:
+        if e.get('parent_action', None):
+            events.append(e)
+        elif e.get('parent', None):
+            parent = e.get('parent')
+            tags = parent.get('with_tags')
+            resources = load_by_tags(tags)
+            for r in resources:
+                parent_action = '{}.{}'.format(r.name, parent['action'])
+                event = {'type' : e['type'],
+                         'state': e['state'],
+                         'depend_action': e['depend_action'],
+                         'parent_action': parent_action
+                         }
+                events.append(event)
+    return events
+
+def parse_events(template_events):
     parsed_events = []
+    events = extend_events(template_events)
     for event in events:
         event_type = event['type']
         parent, parent_action = event['parent_action'].split('.')

solar/solar/test/resource_fixtures/with_location.yaml.tmpl

@@ -0,0 +1,7 @@
+id: simple_multinode
+resources:
+  - id: base_reource
+    location: node1
+    from: {resource_path}
+    values:
+      ip: '10.0.0.3'

solar/solar/test/test_system_log_api.py

@@ -118,7 +118,7 @@ def test_revert_removal():
     assert logitem.diff == [('remove', '', [('a', '9')])]
 
     with mock.patch.object(resource, 'read_meta') as mread:
-        mread.return_value = {'input': {'a': {'schema': 'str!'}}}
+        mread.return_value = {'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'}
         change.revert(logitem.uid)
     resource_obj = resource.load('test1')
     assert resource_obj.args == {'a': '9', 'location_id': '1', 'transports_id': '1'}

solar/solar/test/test_virtual_resource.py

@@ -137,3 +137,24 @@ def test_parse_connection_disable_events():
     }
     connection = vr.parse_connection('ip', 'node1::ip::NO_EVENTS')
     assert correct_connection == connection
+
+def test_setting_location(tmpdir):
+    # XXX: make helper for it
+    base_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'resource_fixtures')
+    vr_node_tmpl_path = os.path.join(base_path, 'nodes.yaml.tmpl')
+    vr_location_tmpl_path = os.path.join(base_path, 'with_location.yaml.tmpl')
+    base_service_path = os.path.join(base_path, 'base_service')
+    node_resource_path = os.path.join(base_path, 'node')
+    with open(vr_node_tmpl_path) as f:
+        vr_data = f.read().format(resource_path=node_resource_path)
+    with open(vr_location_tmpl_path) as f:
+        location_data = f.read().format(resource_path=base_service_path)
+    vr_file = tmpdir.join('nodes.yaml')
+    vr_file.write(vr_data)
+    location_file = tmpdir.join('with_location.yaml')
+    location_file.write(location_data)
+    vr.create('nodes', str(vr_file))
+    resources = vr.create('updates', str(location_file))
+    assert 'location=node1' in resources[0].tags

templates/haproxy.yaml

@@ -0,0 +1,46 @@
+id: haproxy
+
+resources:
+  - id: haproxy_config{{index}}
+    from: resources/haproxy_config
+    location: {{node}}
+    values:
+      ip: '{{node}}::ip'
+      config:protocol:
+        {% for config in service_configs %}
+        - {{config}}::protocol
+        {% endfor %}
+      config:listen_port:
+        {% for config in service_configs %}
+        - {{config}}::listen_port
+        {% endfor %}
+      config:name:
+        {% for config in service_configs %}
+        - {{config}}::name
+        {% endfor %}
+      config:backends:
+        {% for config in service_configs %}
+        - {{config}}::backends
+        {% endfor %}
+
+  - id: haproxy_service{{index}}
+    location: {{node}}
+    from: resources/haproxy_service
+    values:
+      ip: '{{node}}::ip'
+
+events:
+  - type: depends_on
+    parent_action: 'haproxy_service{{index}}.run'
+    state: 'success'
+    depend_action: 'haproxy_config{{index}}.run'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{index}}.run'
+    state: 'success'
+    depend_action: 'haproxy_service{{index}}.apply_config'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{index}}.update'
+    state: 'success'
+    depend_action: 'haproxy_service{{index}}.apply_config'

templates/nodes.yaml

@@ -1,14 +1,27 @@
-id: simple_multinode
+id: simple_riak_with_transports
 resources:
-  - id: node1
+{% for i in range(count|int) %}
+   {% set j = i +1 %}
+  - id: ssh_transport{{j}}
+    from: resources/transport_ssh
+    values:
+      ssh_user: 'vagrant'
+      ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key'
+  - id: transports{{j}}
+    from: resources/transports
+    values:
+      transports:key: ssh_transport{{j}}::ssh_key
+      transports:user: ssh_transport{{j}}::ssh_user
+      transports:port: ssh_transport{{j}}::ssh_port
+      transports:name: ssh_transport{{j}}::name
+  - id: node{{j}}
     from: resources/ro_node
     values:
-      ip: '10.0.0.3'
-      ssh_key: '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key'
-      ssh_user: 'vagrant'
-  - id: node2
-    from: resources/ro_node
-    values:
-      ip: '10.0.0.4'
-      ssh_key: '/vagrant/.vagrant/machines/solar-dev2/virtualbox/private_key'
-      ssh_user: 'vagrant'
+      name: node{{j}}
+      ip: '10.0.0.{{i + 3}}'
+      transports_id: transports{{j}}::transports_id
+  - id: hosts_file{{j}}
+    from: resources/hosts_file
+    location: node{{j}}
+    tags: ['location=node{{j}}']
+{% endfor %}

templates/riak_nodes.yaml

@@ -1,23 +0,0 @@
-id: simple_riak_with_transports
-resources:
-{% for i in range(count|int) %}
-   {% set j = i +1 %}
-  - id: ssh_transport{{j}}
-    from: resources/transport_ssh
-    values:
-      ssh_user: 'vagrant'
-      ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key'
-  - id: transports{{j}}
-    from: resources/transports
-    values:
-      transports:key: ssh_transport{{j}}::ssh_key
-      transports:user: ssh_transport{{j}}::ssh_user
-      transports:port: ssh_transport{{j}}::ssh_port
-      transports:name: ssh_transport{{j}}::name
-  - id: node{{j}}
-    from: resources/ro_node
-    values:
-      name: node{{j}}
-      ip: '10.0.0.{{i + 3}}'
-      transports_id: transports{{j}}::transports_id
-{% endfor %}