From 54bd0ab7e1cc7f66fc909bd5a0739de99d09e031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:36:53 +0200 Subject: [PATCH 01/18] Remove autogenerated tags, add default value --- resources/haproxy_config/meta.yaml | 2 +- resources/haproxy_service/meta.yaml | 2 +- resources/haproxy_service_config/meta.yaml | 4 ++-- resources/hosts_file/meta.yaml | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/resources/haproxy_config/meta.yaml b/resources/haproxy_config/meta.yaml index 5f53e840..10c8d44a 100644 --- a/resources/haproxy_config/meta.yaml +++ b/resources/haproxy_config/meta.yaml @@ -18,4 +18,4 @@ input: # schema: str! # value: -tags: [resources/haproxy, resource/haproxy_config] +tags: [resources=haproxy] diff --git a/resources/haproxy_service/meta.yaml b/resources/haproxy_service/meta.yaml index f6d9aae7..0ea6bf62 100644 --- a/resources/haproxy_service/meta.yaml +++ b/resources/haproxy_service/meta.yaml @@ -12,4 +12,4 @@ input: # schema: str! # value: -tags: [resources/haproxy, resource/haproxy_service] +tags: [resources=haproxy] diff --git a/resources/haproxy_service_config/meta.yaml b/resources/haproxy_service_config/meta.yaml index c6dd6890..a334e0c8 100644 --- a/resources/haproxy_service_config/meta.yaml +++ b/resources/haproxy_service_config/meta.yaml @@ -1,4 +1,4 @@ -id: haproxy_general_config +id: haproxy_service_config handler: none version: 1.0.0 input: @@ -20,4 +20,4 @@ input: # schema: [str] # value: [] -tags: [resources/haproxy, resource/haproxy_general_config] +tags: [resources=haproxy] diff --git a/resources/hosts_file/meta.yaml b/resources/hosts_file/meta.yaml index 255209e2..4fc53abe 100644 --- a/resources/hosts_file/meta.yaml +++ b/resources/hosts_file/meta.yaml @@ -5,3 +5,6 @@ version: 1.0.0 input: hosts: schema: [{name: str!, ip: str!}] + value: [] + +tags: [resource=hosts_file] From 2b9466a315c0912a4b5597e8658c5d8c334063cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:37:21 +0200 Subject: [PATCH 02/18] Add node tag --- resources/ro_node/meta.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/ro_node/meta.yaml b/resources/ro_node/meta.yaml index 3339a0b4..36616e6d 100644 --- a/resources/ro_node/meta.yaml +++ b/resources/ro_node/meta.yaml @@ -1,4 +1,4 @@ -id: mariadb +id: ro_node handler: none version: 1.0.0 actions: @@ -19,3 +19,5 @@ input: schema: str! value: $uuid reverse: True + +tags: [resources=node] From f08bd1472dd6ac1a2b66034c644386379b55de98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:47:53 +0200 Subject: [PATCH 03/18] Merge riak_nodes.yaml and nodes.yaml --- templates/nodes.yaml | 35 ++++++++++++++++++++++++----------- templates/riak_nodes.yaml | 23 ----------------------- 2 files changed, 24 insertions(+), 34 deletions(-) delete mode 100644 templates/riak_nodes.yaml diff --git a/templates/nodes.yaml b/templates/nodes.yaml index fc0261df..3c2e75a9 100644 --- a/templates/nodes.yaml +++ b/templates/nodes.yaml @@ -1,14 +1,27 @@ -id: simple_multinode +id: simple_riak_with_transports resources: - - id: node1 +{% for i in range(count|int) %} + {% set j = i +1 %} + - id: ssh_transport{{j}} + from: resources/transport_ssh + values: + ssh_user: 'vagrant' + ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key' + - id: transports{{j}} + from: resources/transports + values: + transports:key: ssh_transport{{j}}::ssh_key + transports:user: ssh_transport{{j}}::ssh_user + transports:port: ssh_transport{{j}}::ssh_port + transports:name: ssh_transport{{j}}::name + - id: node{{j}} from: resources/ro_node values: - ip: '10.0.0.3' - ssh_key: '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key' - ssh_user: 'vagrant' - - id: node2 - from: resources/ro_node - values: - ip: '10.0.0.4' - ssh_key: '/vagrant/.vagrant/machines/solar-dev2/virtualbox/private_key' - ssh_user: 'vagrant' + name: node{{j}} + ip: '10.0.0.{{i + 3}}' + transports_id: transports{{j}}::transports_id + - id: hosts_file{{j}} + from: resources/hosts_file + location: node{{j}} + tags: ['location=node{{j}}'] +{% endfor %} diff --git a/templates/riak_nodes.yaml b/templates/riak_nodes.yaml deleted file mode 100644 index 6d93c1eb..00000000 --- a/templates/riak_nodes.yaml +++ /dev/null @@ -1,23 +0,0 @@ -id: simple_riak_with_transports -resources: -{% for i in range(count|int) %} - {% set j = i +1 %} - - id: ssh_transport{{j}} - from: resources/transport_ssh - values: - ssh_user: 'vagrant' - ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key' - - id: transports{{j}} - from: resources/transports - values: - transports:key: ssh_transport{{j}}::ssh_key - transports:user: ssh_transport{{j}}::ssh_user - transports:port: ssh_transport{{j}}::ssh_port - transports:name: ssh_transport{{j}}::name - - id: node{{j}} - from: resources/ro_node - values: - name: node{{j}} - ip: '10.0.0.{{i + 3}}' - transports_id: transports{{j}}::transports_id -{% endfor %} From 3d6f48ffcfc538b9acc3a6547ab0310be79a0223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:50:32 +0200 Subject: [PATCH 04/18] Fix quotes --- resources/hosts_file/actions/run.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/hosts_file/actions/run.yaml b/resources/hosts_file/actions/run.yaml index b6f94195..8ad5f23f 100644 --- a/resources/hosts_file/actions/run.yaml +++ b/resources/hosts_file/actions/run.yaml @@ -5,7 +5,7 @@ - name: Create hosts entries for {{val['name']}} => {{val['ip']}} lineinfile: dest: /etc/hosts - regexp: '.*{{val['name']}}$' - line: '{{val['ip']}} {{val['name']}}' + regexp: ".*{{val['name']}}$" + line: "{{val['ip']}} {{val['name']}}" state: present {% endfor %} From f3acc3fc15fa733c2dc9efb6fc62f5dcb56c112f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:50:50 +0200 Subject: [PATCH 05/18] Add update action for hosts_file resource --- resources/hosts_file/actions/update.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 resources/hosts_file/actions/update.yaml diff --git a/resources/hosts_file/actions/update.yaml b/resources/hosts_file/actions/update.yaml new file mode 100644 index 00000000..8ad5f23f --- /dev/null +++ b/resources/hosts_file/actions/update.yaml @@ -0,0 +1,11 @@ +- hosts: [{{host}}] + sudo: yes + tasks: + {% for val in hosts %} + - name: Create hosts entries for {{val['name']}} => {{val['ip']}} + lineinfile: + dest: /etc/hosts + regexp: ".*{{val['name']}}$" + line: "{{val['ip']}} {{val['name']}}" + state: present + {% endfor %} From 2930f18eaa968e7db724ff0f4000cbbcafa94dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:53:31 +0200 Subject: [PATCH 06/18] Restore tags support in resources --- solar/solar/core/resource/__init__.py | 2 +- solar/solar/core/resource/resource.py | 37 ++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/solar/solar/core/resource/__init__.py b/solar/solar/core/resource/__init__.py index 5736260e..bcea2dd8 100644 --- a/solar/solar/core/resource/__init__.py +++ b/solar/solar/core/resource/__init__.py @@ -12,4 +12,4 @@ # License for the specific language governing permissions and limitations # under the License. -from .resource import Resource, load, load_all, validate_resources +from .resource import Resource, load, load_all, validate_resources, filter_resources diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 214a03dd..11744228 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -59,7 +59,12 @@ class Resource(object): metadata = deepcopy(self._metadata) self.base_path = base_path - self.tags = tags or [] + if tags is None: + tags = [] + m_tags = metadata.get('tags', []) + tags.extend(m_tags) + tags.append('resource={}'.format(metadata['id'])) + self.virtual_resource = virtual_resource inputs = metadata.get('input', {}) @@ -75,7 +80,8 @@ class Resource(object): 'handler': metadata.get('handler', ''), 'puppet_module': metadata.get('puppet_module', ''), 'version': metadata.get('version', ''), - 'meta_inputs': inputs + 'meta_inputs': inputs, + 'tags': tags }) self.db_obj.state = RESOURCE_STATE.created.name @@ -89,8 +95,6 @@ class Resource(object): self.db_obj = resource_db self.name = resource_db.name self.base_path = resource_db.base_path - # TODO: tags - self.tags = [] self.virtual_resource = None def auto_extend_inputs(self, inputs): @@ -228,6 +232,28 @@ class Resource(object): def load_commited(self): return orm.DBCommitedState.get_or_create(self.name) + # XXX: Make tags faster, use db for it + def has_tags(self, tags): + db_tags = self.db_obj.tags + found_tags = [] + for tag in tags: + if tag in db_tags: + found_tags.append(tag) + return found_tags == tags + + def add_tag(self, tag): + if tag not in self.db_obj.tags: + self.db_obj.tags.append(tag) + self.db_obj.save() + + def remove_tag(self, tag): + if tag in self.db_obj.tags: + self.db_obj.tags.remove(tag) + self.db_obj.save() + + def get_tags(self, tag): + return self.db_obj.tags +>>>>>>> Restore tags support in resources def load(name): r = orm.DBResource.load(name) @@ -242,6 +268,9 @@ def load(name): def load_all(): return [Resource(r) for r in orm.DBResource.load_all()] +def filter_resources(tags): + # XXX: should it be a class method? + return [r for r in load_all() if r.has_tags(tags)] def validate_resources(): resources = load_all() From 094f6bb9c9370977e75cc3a300ef7962b80c3f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 12:58:56 +0200 Subject: [PATCH 07/18] Fix using tags in show command and log in tag --- solar/solar/cli/resource.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py index e4986ef7..428076b7 100644 --- a/solar/solar/cli/resource.py +++ b/solar/solar/cli/resource.py @@ -136,19 +136,14 @@ def create(args, base_path, name): @click.option('--json', default=False, is_flag=True) @click.option('--color', default=True, is_flag=True) def show(**kwargs): - resources = [] + resources = sresource.load_all() - for res in sresource.load_all(): - show = True - if kwargs['tag']: - if kwargs['tag'] not in res.tags: - show = False - if kwargs['name']: - if res.name != kwargs['name']: - show = False - - if show: - resources.append(res) + if kwargs['tag']: + tags = kwargs['tag'].split(' ') + resources = filter(lambda r: r.has_tags(tags), resources) + if kwargs['name']: + name = kwargs['name'] + resources = filter(lambda r: r.name == name, resources) echo = click.echo_via_pager if kwargs['json']: @@ -169,13 +164,13 @@ def show(**kwargs): @click.argument('tag_name') @click.option('--add/--delete', default=True) def tag(add, tag_name, resource_name): - click.echo('Tag {} with {} {}'.format(resource_name, tag_name, add)) r = sresource.load(resource_name) if add: r.add_tag(tag_name) + click.echo('Tag {} added to {}'.format(tag_name, resource_name)) else: r.remove_tag(tag_name) - # TODO: the above functions should save resource automatically to the DB + click.echo('Tag {} removed from {}'.format(tag_name, resource_name)) @resource.command() @click.argument('name') From 09077fc4873adadafbd574100ee31a3e38b2a4b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 14:38:42 +0200 Subject: [PATCH 08/18] Add support for tags in Virtual Resources - resouces are now taged propperly - with_tags selector may be used to make connections for more than one resource --- solar/solar/core/resource/virtual_resource.py | 61 +++++++++++++++---- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/solar/solar/core/resource/virtual_resource.py b/solar/solar/core/resource/virtual_resource.py index a9359050..a8378be6 100644 --- a/solar/solar/core/resource/virtual_resource.py +++ b/solar/solar/core/resource/virtual_resource.py @@ -21,13 +21,14 @@ from jinja2 import Template, Environment, meta from solar.core import provider from solar.core import signals +from solar.core.log import log from solar.core.resource import load as load_resource -from solar.core.resource import Resource +from solar.core.resource import Resource, filter_resources from solar.events.api import add_event from solar.events.controls import React, Dep -def create(name, base_path, args=None, virtual_resource=None): +def create(name, base_path, args=None, tags=None, virtual_resource=None): args = args or {} if isinstance(base_path, provider.BaseProvider): base_path = base_path.directory @@ -40,18 +41,19 @@ def create(name, base_path, args=None, virtual_resource=None): if is_virtual(base_path): template = _compile_file(name, base_path, args) yaml_template = yaml.load(StringIO(template)) - rs = create_virtual_resource(name, yaml_template) + rs = create_virtual_resource(name, yaml_template, tags) else: r = create_resource(name, base_path, args=args, + tags=tags, virtual_resource=virtual_resource) rs = [r] return rs -def create_resource(name, base_path, args=None, virtual_resource=None): +def create_resource(name, base_path, args=None, tags=None, virtual_resource=None): args = args or {} if isinstance(base_path, provider.BaseProvider): base_path = base_path.directory @@ -59,17 +61,17 @@ def create_resource(name, base_path, args=None, virtual_resource=None): # List args init with empty list. Elements will be added later args = {key: (value if not isinstance(value, list) else []) for key, value in args.items()} r = Resource( - name, base_path, args=args, tags=[], virtual_resource=virtual_resource + name, base_path, args=args, tags=tags, virtual_resource=virtual_resource ) return r -def create_virtual_resource(vr_name, template): +def create_virtual_resource(vr_name, template, tags=None): template_resources = template.get('resources', []) template_events = template.get('events', []) resources_to_update = template.get('updates', []) - created_resources = create_resources(template_resources) + created_resources = create_resources(template_resources, tags=tags) events = parse_events(template_events) for event in events: add_event(event) @@ -111,27 +113,43 @@ def is_virtual(path): return os.path.isfile(path) -def create_resources(resources): +def create_resources(resources, tags=None): created_resources = [] cwd = os.getcwd() for r in resources: resource_name = r['id'] - args = r['values'] + args = r.get('values', {}) node = r.get('location', None) from_path = r.get('from', None) + tags = r.get('tags', []) base_path = os.path.join(cwd, from_path) - new_resources = create(resource_name, base_path) + new_resources = create(resource_name, base_path, tags=tags) created_resources += new_resources if not is_virtual(base_path): if node: node = load_resource(node) r = load_resource(resource_name) signals.connect(node, r, {}) + r.add_tag('location={}'.format(node.name)) update_inputs(resource_name, args) return created_resources -def update_resources(resources): +def update_resources(template_resources): + resources = [] + for r in template_resources: + if r.get('id'): + resources.append(r) + if r.get('with_tags'): + tags = r.get('with_tags') + filtered = filter_resources(tags) + for f in filtered: + r = {'id': f.name, + 'values': r['values']} + resources.append(r) + log.debug('Resource {} for tags {} found'.format(r, tags)) + if not filtered: + log.debug('Warrning: no resources with tags: {}'.format(tags)) for r in resources: resource_name = r['id'] args = r['values'] @@ -151,7 +169,26 @@ def update_inputs(child, args): child.update(assignments) -def parse_events(events): +def parse_events(template_events): + #extend events + events = [] + for e in template_events: + if e.get('parent_action', None): + events.append(e) + elif e.get('parent', None): + parent = e.get('parent') + tags = parent.get('with_tags') + resources = filter_resources(tags) + for r in resources: + parent_action = '{}.{}'.format(r.name, parent['action']) + event = {'type' : e['type'], + 'state': e['state'], + 'depend_action': e['depend_action'], + 'parent_action': parent_action + } + events.append(event) + + ### parsed_events = [] for event in events: event_type = event['type'] From ebb3048a805b38543a38ef48c5f891bc7906bead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 14:40:41 +0200 Subject: [PATCH 09/18] Split riak cluster into smaller pieces --- examples/riak/haproxy_riak_config.yaml | 50 +++++++++++++++++++++ examples/riak/riak_service.yaml | 61 ++++++++++++++++++++++++++ templates/haproxy.yaml | 46 +++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 examples/riak/haproxy_riak_config.yaml create mode 100644 examples/riak/riak_service.yaml create mode 100644 templates/haproxy.yaml diff --git a/examples/riak/haproxy_riak_config.yaml b/examples/riak/haproxy_riak_config.yaml new file mode 100644 index 00000000..2a044693 --- /dev/null +++ b/examples/riak/haproxy_riak_config.yaml @@ -0,0 +1,50 @@ +id: haproxy_riak_config + +resources: + - id: haproxy_riak_config_http + from: resources/haproxy_service_config + tags: ['service=riak', 'protocol=http'] + values: + listen_port: {{http_listen_port}} + protocol: 'http' + name: 'riak_haproxy_http' + backends:server: + {% for riak in riaks %} + - {{riak}}::riak_hostname + {% endfor %} + backends:port: + {% for riak in riaks %} + - {{riak}}::riak_port_http + {% endfor %} + + - id: haproxy_riak_config_pb + from: resources/haproxy_service_config + tags: ['service=riak', 'protocol=tcp'] + values: + listen_port: {{pb_listen_port}} + protocol: 'tcp' + name: 'riak_haproxy_pb' + backends:server: + {% for riak in riaks %} + - {{riak}}::riak_hostname + {% endfor %} + backends:port: + {% for riak in riaks %} + - {{riak}}::riak_port_pb + {% endfor %} + +updates: + - with_tags: ['resource=haproxy_config'] + values: + config:protocol: + - haproxy_riak_config_http::protocol + - haproxy_riak_config_pb::protocol + config:listen_port: + - haproxy_riak_config_http::listen_port + - haproxy_riak_config_pb::listen_port + config:name: + - haproxy_riak_config_http::name + - haproxy_riak_config_pb::name + config:backends: + - haproxy_riak_config_http::backends + - haproxy_riak_config_pb::backends diff --git a/examples/riak/riak_service.yaml b/examples/riak/riak_service.yaml new file mode 100644 index 00000000..acf931d2 --- /dev/null +++ b/examples/riak/riak_service.yaml @@ -0,0 +1,61 @@ +id: riak_service + +resources: + - id: riak_service{{index}} + from: resources/riak_node + location: {{node}} + values: + riak_self_name: riak{{index}} + riak_hostname: riak_server{{index}}.solar + riak_name: riak{{index}}@riak_server{{index}}.solar + {% if join_to %} + join_to: {{join_to}}::riak_name + {% endif %} + ip: {{node}}::ip + +updates: + - with_tags: ['resource=hosts_file'] + values: + hosts:name: + - riak_service{{index}}::riak_hostname::NO_EVENTS + hosts:ip: + - riak_service{{index}}::ip::NO_EVENTS + + - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=http'] + values: + backends:server: + - riak_service{{index}}::riak_hostname + backends:port: + - riak_service{{index}}::riak_port_http + + - with_tags: ['resource=haproxy_service_config', 'service=riak', 'protocol=tcp'] + values: + backends:server: + - riak_service{{index}}::riak_hostname + backends:port: + - riak_service{{index}}::riak_port_pb + +events: + - type: depends_on + parent: + with_tags: ['resource=hosts_file', 'location={{node}}'] + action: run + state: success + depend_action: riak_service{{index}}.run + +{% if join_to %} + - type: react_on + parent_action: riak_service{{index}}.run + state: success + depend_action: riak_service{{index}}.join + + - type: react_on + parent_action: riak_service{{index}}.leave + state: success + depend_action: riak_service{{index}}.join + + - type: react_on + parent_action: riak_service{{index}}.join + state: success + depend_action: {{join_to}}.commit +{% endif %} diff --git a/templates/haproxy.yaml b/templates/haproxy.yaml new file mode 100644 index 00000000..9372b3d8 --- /dev/null +++ b/templates/haproxy.yaml @@ -0,0 +1,46 @@ +id: haproxy + +resources: + - id: haproxy_config{{index}} + from: resources/haproxy_config + location: {{node}} + values: + ip: '{{node}}::ip' + config:protocol: + {% for config in service_configs %} + - {{config}}::protocol + {% endfor %} + config:listen_port: + {% for config in service_configs %} + - {{config}}::listen_port + {% endfor %} + config:name: + {% for config in service_configs %} + - {{config}}::name + {% endfor %} + config:backends: + {% for config in service_configs %} + - {{config}}::backends + {% endfor %} + + - id: haproxy_service{{index}} + location: {{node}} + from: resources/haproxy_service + values: + ip: '{{node}}::ip' + +events: + - type: depends_on + parent_action: 'haproxy_service{{index}}.run' + state: 'success' + depend_action: 'haproxy_config{{index}}.run' + + - type: react_on + parent_action: 'haproxy_config{{index}}.run' + state: 'success' + depend_action: 'haproxy_service{{index}}.apply_config' + + - type: react_on + parent_action: 'haproxy_config{{index}}.update' + state: 'success' + depend_action: 'haproxy_service{{index}}.apply_config' From b5212fc2663b30f42c9e94232623192bc2adea6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 15:04:38 +0200 Subject: [PATCH 10/18] Update Virtual Resource for riak cluster --- examples/riak/riak_cluster.yaml | 160 +++++++------------------------- 1 file changed, 36 insertions(+), 124 deletions(-) diff --git a/examples/riak/riak_cluster.yaml b/examples/riak/riak_cluster.yaml index 4fc93e91..98143d1f 100644 --- a/examples/riak/riak_cluster.yaml +++ b/examples/riak/riak_cluster.yaml @@ -1,139 +1,51 @@ -id: riak_node +id: riak_cluster resources: - - id: riak_service0 - from: resources/riak_node - location: {{nodes[0]}} + - id: riak_service1 + from: examples/riak/riak_service.yaml values: - riak_self_name: 'riak0' - riak_hostname: 'riak_server0.solar' - riak_name: 'riak0@riak_server0.solar' - ip: '{{nodes[0]}}::ip' + node: {{nodes[0]}} + index: 1 + join_to: '' -{% for i in range(1, idx|int) %} - - id: riak_service{{i}} - from: resources/riak_node - location: {{nodes[i]}} + - id: riak_service2 + from: examples/riak/riak_service.yaml values: - riak_self_name: 'riak{{i}}' - riak_hostname: 'riak_server{{i}}.solar' - riak_name: 'riak{{i}}@riak_server{{i}}.solar' - join_to: 'riak_service0::riak_name' - ip: '{{nodes[i]}}::ip' -{% endfor %} + node: {{nodes[1]}} + index: 2 + join_to: riak_service1 -{% for i in range(idx|int) %} - - id: hosts_file{{i}} - from: resources/hosts_file - location: {{nodes[i]}} + - id: riak_service3 + from: examples/riak/riak_service.yaml values: - hosts:name: - {% for j in range(idx|int) %} - - riak_service{{j}}::riak_hostname::NO_EVENTS - {% endfor %} - hosts:ip: - {% for j in range(idx|int) %} - - riak_service{{j}}::ip::NO_EVENTS - {% endfor %} -{% endfor %} + node: {{nodes[2]}} + index: 3 + join_to: riak_service1 - - id: haproxy_riak_config_http - from: resources/haproxy_service_config + - id: haproxy_riak_config + from: examples/riak/haproxy_riak_config.yaml values: - listen_port: 8098 - protocol: 'http' - name: 'riak_haproxy_http' - backends:server: - {% for j in range(idx|int) %} - - riak_service{{j}}::riak_hostname - {% endfor %} - backends:port: - {% for j in range(idx|int) %} - - riak_service{{j}}::riak_port_http - {% endfor %} + http_listen_port: 8098 + pb_listen_port: 8087 + riaks: ['riak_service1', 'riak_service2', 'riak_service3'] - - id: haproxy_riak_config_pb - from: resources/haproxy_service_config + - id: haproxy1 + from: templates/haproxy.yaml values: - listen_port: 8087 - protocol: 'tcp' - name: 'riak_haproxy_pb' - backends:server: - {% for j in range(idx|int) %} - - riak_service{{j}}::riak_hostname - {% endfor %} - backends:port: - {% for j in range(idx|int) %} - - riak_service{{j}}::riak_port_pb - {% endfor %} + node: {{nodes[0]}} + service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http'] + index: 1 -{% for i in range(idx|int) %} - - id: haproxy_config{{i}} - from: resources/haproxy_config - location: {{nodes[i]}} + - id: haproxy2 + from: templates/haproxy.yaml values: - config:protocol: - - haproxy_riak_config_http::protocol - - haproxy_riak_config_pb::protocol - config:listen_port: - - haproxy_riak_config_http::listen_port - - haproxy_riak_config_pb::listen_port - config:name: - - haproxy_riak_config_http::name - - haproxy_riak_config_pb::name - config:backends: - - haproxy_riak_config_http::backends - - haproxy_riak_config_pb::backends - ip: '{{nodes[i]}}::ip' -{% endfor %} + node: {{nodes[1]}} + service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http'] + index: 2 -{% for i in range(idx|int) %} - - id: haproxy_service{{i}} - location: {{nodes[i]}} - from: resources/haproxy_service + - id: haproxy3 + from: templates/haproxy.yaml values: - ip: '{{nodes[i]}}::ip' -{% endfor %} - - -events: -{% for i in range(idx|int) %} - - type: depends_on - parent_action: 'hosts_file{{i}}.run' - state: 'success' - depend_action: 'riak_service{{i}}.run' -{% endfor %} - -{% for i in range(1, idx|int) %} - - type: react_on - parent_action: 'riak_service{{i}}.run' - state: 'success' - depend_action: 'riak_service{{i}}.join' - - - type: react_on - parent_action: 'riak_service{{i}}.leave' - state: 'success' - depend_action: 'riak_service{{i}}.join' - - - type: react_on - parent_action: 'riak_service{{i}}.join' - state: 'success' - depend_action: 'riak_service0.commit' -{% endfor %} - -{% for i in range(1, idx|int) %} - - type: depends_on - parent_action: 'haproxy_service{{i}}.run' - state: 'success' - depend_action: 'haproxy_config{{i}}.run' - - - type: react_on - parent_action: 'haproxy_config{{i}}.run' - state: 'success' - depend_action: 'haproxy_service{{i}}.apply_config' - - - type: react_on - parent_action: 'haproxy_config{{i}}.update' - state: 'success' - depend_action: 'haproxy_service{{i}}.apply_config' -{% endfor %} + node: {{nodes[2]}} + service_configs: ['haproxy_riak_config_pb', 'haproxy_riak_config_http'] + index: 3 From fb3a799ebaf911ca9a88ba256c1cd4213eb0e3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 15:20:32 +0200 Subject: [PATCH 11/18] Fix test --- solar/solar/core/resource/resource.py | 2 +- solar/solar/test/test_resource.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 11744228..e5436972 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -251,7 +251,7 @@ class Resource(object): self.db_obj.tags.remove(tag) self.db_obj.save() - def get_tags(self, tag): + def get_tags(self): return self.db_obj.tags >>>>>>> Restore tags support in resources diff --git a/solar/solar/test/test_resource.py b/solar/solar/test/test_resource.py index a3af0256..27aeb581 100644 --- a/solar/solar/test/test_resource.py +++ b/solar/solar/test/test_resource.py @@ -92,7 +92,7 @@ input: sample_l = resource.load('sample') self.assertDictEqual(sample.args, sample_l.args) - self.assertListEqual(sample.tags, sample_l.tags) + self.assertListEqual(sample.get_tags(), sample_l.get_tags()) def test_removal(self): """Test that connection removed with resource.""" From 0cc436f55d9706588378e178548dd9542fa0c9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 15:21:56 +0200 Subject: [PATCH 12/18] Pass args to create function It is required when Virtual Resource is created --- solar/solar/core/resource/virtual_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/core/resource/virtual_resource.py b/solar/solar/core/resource/virtual_resource.py index a8378be6..7afbfd02 100644 --- a/solar/solar/core/resource/virtual_resource.py +++ b/solar/solar/core/resource/virtual_resource.py @@ -123,7 +123,7 @@ def create_resources(resources, tags=None): from_path = r.get('from', None) tags = r.get('tags', []) base_path = os.path.join(cwd, from_path) - new_resources = create(resource_name, base_path, tags=tags) + new_resources = create(resource_name, base_path, args=args, tags=tags) created_resources += new_resources if not is_virtual(base_path): if node: From 22c6bb98494cf53ad194abc16dd8fc0a547cde19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 19:01:51 +0200 Subject: [PATCH 13/18] Remove `>>>>` after merge --- solar/solar/core/resource/resource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index e5436972..01cd29a0 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -253,7 +253,6 @@ class Resource(object): def get_tags(self): return self.db_obj.tags ->>>>>>> Restore tags support in resources def load(name): r = orm.DBResource.load(name) From 382f6cf779fc9cbdc60f5708a9d594c799629615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 22:19:06 +0200 Subject: [PATCH 14/18] Use function to get tags --- solar/solar/system_log/operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/system_log/operations.py b/solar/solar/system_log/operations.py index 4a692881..600e7e8a 100644 --- a/solar/solar/system_log/operations.py +++ b/solar/solar/system_log/operations.py @@ -44,7 +44,7 @@ def move_to_commited(log_action, *args, **kwargs): resource_obj.set_operational() commited.state = resource.RESOURCE_STATE.operational.name commited.inputs = patch(item.diff, commited.inputs) - commited.tags = resource_obj.tags + commited.tags = resource_obj.get_tags() sorted_connections = sorted(commited.connections) commited.connections = patch(item.signals_diff, sorted_connections) commited.base_path = item.base_path From e7ae77d870dad9758a9b6f7553368a97a893795a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 22:19:22 +0200 Subject: [PATCH 15/18] id is mandatory in metadata --- solar/solar/test/test_system_log_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/test/test_system_log_api.py b/solar/solar/test/test_system_log_api.py index a467060c..9117e2c8 100644 --- a/solar/solar/test/test_system_log_api.py +++ b/solar/solar/test/test_system_log_api.py @@ -118,7 +118,7 @@ def test_revert_removal(): assert logitem.diff == [('remove', '', [('a', '9')])] with mock.patch.object(resource, 'read_meta') as mread: - mread.return_value = {'input': {'a': {'schema': 'str!'}}} + mread.return_value = {'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'} change.revert(logitem.uid) resource_obj = resource.load('test1') assert resource_obj.args == {'a': '9', 'location_id': '1', 'transports_id': '1'} From a0adbbe0b473dc75546e14559c06aba5ea0d36b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 23:07:39 +0200 Subject: [PATCH 16/18] Add test for setting a location --- solar/solar/core/resource/virtual_resource.py | 2 +- .../resource_fixtures/with_location.yaml.tmpl | 7 +++++++ solar/solar/test/test_virtual_resource.py | 21 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 solar/solar/test/resource_fixtures/with_location.yaml.tmpl diff --git a/solar/solar/core/resource/virtual_resource.py b/solar/solar/core/resource/virtual_resource.py index 7afbfd02..93dd10d5 100644 --- a/solar/solar/core/resource/virtual_resource.py +++ b/solar/solar/core/resource/virtual_resource.py @@ -128,7 +128,7 @@ def create_resources(resources, tags=None): if not is_virtual(base_path): if node: node = load_resource(node) - r = load_resource(resource_name) + r = new_resources[0] signals.connect(node, r, {}) r.add_tag('location={}'.format(node.name)) update_inputs(resource_name, args) diff --git a/solar/solar/test/resource_fixtures/with_location.yaml.tmpl b/solar/solar/test/resource_fixtures/with_location.yaml.tmpl new file mode 100644 index 00000000..bd78b227 --- /dev/null +++ b/solar/solar/test/resource_fixtures/with_location.yaml.tmpl @@ -0,0 +1,7 @@ +id: simple_multinode +resources: + - id: base_reource + location: node1 + from: {resource_path} + values: + ip: '10.0.0.3' diff --git a/solar/solar/test/test_virtual_resource.py b/solar/solar/test/test_virtual_resource.py index e9d47cd1..babb6f80 100644 --- a/solar/solar/test/test_virtual_resource.py +++ b/solar/solar/test/test_virtual_resource.py @@ -137,3 +137,24 @@ def test_parse_connection_disable_events(): } connection = vr.parse_connection('ip', 'node1::ip::NO_EVENTS') assert correct_connection == connection + +def test_setting_location(tmpdir): + # XXX: make helper for it + base_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'resource_fixtures') + vr_node_tmpl_path = os.path.join(base_path, 'nodes.yaml.tmpl') + vr_location_tmpl_path = os.path.join(base_path, 'with_location.yaml.tmpl') + base_service_path = os.path.join(base_path, 'base_service') + node_resource_path = os.path.join(base_path, 'node') + with open(vr_node_tmpl_path) as f: + vr_data = f.read().format(resource_path=node_resource_path) + with open(vr_location_tmpl_path) as f: + location_data = f.read().format(resource_path=base_service_path) + vr_file = tmpdir.join('nodes.yaml') + vr_file.write(vr_data) + location_file = tmpdir.join('with_location.yaml') + location_file.write(location_data) + vr.create('nodes', str(vr_file)) + resources = vr.create('updates', str(location_file)) + assert 'location=node1' in resources[0].get_tags() From eb9937b3c71c9d020e9f77d9c63bc19c361247dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Tue, 29 Sep 2015 23:25:35 +0200 Subject: [PATCH 17/18] Move extracting resources from tags to seperate functions --- solar/solar/core/resource/virtual_resource.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/solar/solar/core/resource/virtual_resource.py b/solar/solar/core/resource/virtual_resource.py index 93dd10d5..25de851c 100644 --- a/solar/solar/core/resource/virtual_resource.py +++ b/solar/solar/core/resource/virtual_resource.py @@ -135,7 +135,7 @@ def create_resources(resources, tags=None): return created_resources -def update_resources(template_resources): +def extend_resources(template_resources): resources = [] for r in template_resources: if r.get('id'): @@ -150,6 +150,10 @@ def update_resources(template_resources): log.debug('Resource {} for tags {} found'.format(r, tags)) if not filtered: log.debug('Warrning: no resources with tags: {}'.format(tags)) + return resources + +def update_resources(template_resources): + resources = extend_resources(template_resources) for r in resources: resource_name = r['id'] args = r['values'] @@ -169,8 +173,7 @@ def update_inputs(child, args): child.update(assignments) -def parse_events(template_events): - #extend events +def extend_events(template_events): events = [] for e in template_events: if e.get('parent_action', None): @@ -187,9 +190,11 @@ def parse_events(template_events): 'parent_action': parent_action } events.append(event) + return events - ### +def parse_events(template_events): parsed_events = [] + events = extend_events(template_events) for event in events: event_type = event['type'] parent, parent_action = event['parent_action'].split('.') From 38dc59f756c4c9c69d9dd0c5c8f66fce0d1d7227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Wed, 30 Sep 2015 15:08:04 +0200 Subject: [PATCH 18/18] Always convert to set --- solar/solar/core/resource/resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 30ce831e..a26b3844 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -263,6 +263,7 @@ def load_all(): return [Resource(r) for r in orm.DBResource.load_all()] def load_by_tags(tags): + tags = set(tags) return [Resource(r) for r in orm.DBResource.load_all() if tags.issubset(set(r.tags))]