From 6dd3294bfdab22f6ad03bce770bfdcd536e3bd5c Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Fri, 18 Sep 2015 11:36:08 +0200 Subject: [PATCH] Riaks: fix template example, small riaks.py fixes --- examples/riak/riaks-template.py | 262 +++++++++++++++++++------------- examples/riak/riaks.py | 28 ++-- solar/solar/core/signals.py | 2 +- solar/solar/template.py | 51 ++++--- 4 files changed, 205 insertions(+), 138 deletions(-) diff --git a/examples/riak/riaks-template.py b/examples/riak/riaks-template.py index 751f9832..97087397 100644 --- a/examples/riak/riaks-template.py +++ b/examples/riak/riaks-template.py @@ -4,116 +4,172 @@ # work here, for most up-to-date version see example-riaks.py # This is just a demo of the template language of Solar +import click +import sys + +from solar.core import resource from solar.interfaces.db import get_db -from solar.core import signals - -db = get_db() -db.clear() -signals.Connections.clear() - from solar import template -nodes = template.nodes_from('templates/riak_nodes.yaml') - -riak_services = nodes.on_each( - 'resources/riak_node', - { - 'riak_self_name': 'riak{num}', - 'riak_hostname': 'riak_server{num}.solar', - 'riak_name': 'riak{num}@riak_server{num}.solar', -}) - -slave_riak_services = riak_services.tail() - -riak_services.take(0).connect_list( - slave_riak_services, - { - 'riak_name': 'join_to', - } -) - -hosts_files = nodes.on_each('resources/hosts_file') - -riak_services.connect_list_to_each( - hosts_files, - { - 'ip': 'hosts_ips', - 'riak_hostname': 'hosts_names', - }, - events=False -) +db = get_db() -hosts_files.add_deps('run/success', riak_services, 'run') -slave_riak_services.add_reacts('run/success', slave_riak_services, 'join') -slave_riak_services.add_reacts('leave/success', slave_riak_services, 'join') -slave_riak_services.add_react('run/success', riak_services.take(0), 'commit') +def setup_riak(): + db.clear() + + nodes = template.nodes_from('templates/riak_nodes.yaml') + + riak_services = nodes.on_each( + 'resources/riak_node', + args={ + 'riak_self_name': 'riak{num}', + 'riak_hostname': 'riak_server{num}.solar', + 'riak_name': 'riak{num}@riak_server{num}.solar', + } + ) + + slave_riak_services = riak_services.tail() + + riak_services.take(0).connect_list( + slave_riak_services, + mapping={ + 'riak_name': 'join_to', + } + ) + + hosts_files = nodes.on_each('resources/hosts_file') + + riak_services.connect_list_to_each( + hosts_files, + mapping={ + 'ip': 'hosts:ip', + 'riak_hostname': 'hosts:name', + }, + events=False + ) + + errors = resource.validate_resources() + for r, error in errors: + click.echo('ERROR: %s: %s' % (r.name, error)) + + if errors: + click.echo("ERRORS") + sys.exit(1) + + hosts_files.add_deps('run/success', riak_services, 'run') + slave_riak_services.add_reacts('run/success', slave_riak_services, 'join') + slave_riak_services.add_reacts('leave/success', slave_riak_services, 'join') + slave_riak_services.add_react('run/success', riak_services.take(0), 'commit') -haproxy_services = nodes.on_each( - 'resources/haproxy_service' -) -haproxy_configs = nodes.on_each( - 'resources/haproxy_config' -) -haproxy_service_configs_http = nodes.on_each( - 'resources/haproxy_service_config', - { - 'listen_port': 8098, - 'protocol': 'http', - } -) -haproxy_service_configs_pb = nodes.on_each( - 'resources/haproxy_service_config', - { - 'listen_port': 8087, - 'protocol': 'tcp', - } -) +def setup_haproxies(): + # TODO: VR loading needs to be supported, then we can do something like + # nodes = template.load('nodes') -riak_services.connect_list_to_each( - haproxy_service_configs_http, - { - 'riak_hostname': 'servers', - 'riak_port_http': 'ports', - } -) -riak_services.connect_list_to_each( - haproxy_service_configs_pb, - { - 'riak_hostname': 'servers', - 'riak_port_pb': 'ports', - } -) -haproxy_service_configs_http.connect_list( - haproxy_configs, - { - 'protocol': 'configs_protocols', - 'listen_port': 'listen_ports', - 'name': 'configs_names', - 'servers': 'configs', - 'ports': 'configs_ports', - } -) -haproxy_service_configs_pb.connect_list( - haproxy_configs, - { - 'protocol': 'configs_protocols', - 'listen_port': 'listen_ports', - 'name': 'configs_names', - 'servers': 'configs', - 'ports': 'configs_ports', - } -) -haproxy_configs.connect_list( - haproxy_services, - { - 'listen_ports': 'ports', - } -) + nodes = template.ResourceListTemplate([ + resource.load('node1'), + resource.load('node2'), + resource.load('node3'), + ]) + riak_services = template.ResourceListTemplate([ + resource.load('riak_node-0'), + resource.load('riak_node-1'), + resource.load('riak_node-2'), + ]) -nodes.add_reacts('run/success', haproxy_services, 'install') -haproxy_services.add_deps('install/success', haproxy_configs, 'run') -haproxy_configs.add_reacts('run/success', haproxy_services, 'run') -haproxy_configs.add_reacts('update/success', haproxy_services, 'update') + haproxy_services = nodes.on_each( + 'resources/haproxy_service' + ) + haproxy_configs = nodes.on_each( + 'resources/haproxy_config' + ) + haproxy_service_configs_http = nodes.on_each( + 'resources/haproxy_service_config', + { + 'listen_port': 8098, + 'protocol': 'http', + } + ) + haproxy_service_configs_pb = nodes.on_each( + 'resources/haproxy_service_config', + { + 'listen_port': 8087, + 'protocol': 'tcp', + } + ) + + riak_services.connect_list_to_each( + haproxy_service_configs_http, + { + 'riak_hostname': 'backends:server', + 'riak_port_http': 'backends:port', + } + ) + riak_services.connect_list_to_each( + haproxy_service_configs_pb, + { + 'riak_hostname': 'backends:server', + 'riak_port_pb': 'backends:port', + } + ) + haproxy_service_configs_http.connect_list( + haproxy_configs, + { + 'backends': 'config:backends', + 'listen_port': 'config:listen_port', + 'protocol': 'config:protocol', + 'name': 'config:name', + } + ) + haproxy_service_configs_pb.connect_list( + haproxy_configs, + { + 'backends': 'config:backends', + 'listen_port': 'config:listen_port', + 'protocol': 'config:protocol', + 'name': 'config:name', + } + ) + + #nodes.add_reacts('run/success', haproxy_services, 'install') + haproxy_services.add_deps('run/success', haproxy_configs, 'run') + haproxy_configs.add_reacts('run/success', haproxy_services, 'apply_config') + haproxy_configs.add_reacts('update/success', haproxy_services, 'apply_config') + + errors = resource.validate_resources() + for r, error in errors: + click.echo('ERROR: %s: %s' % (r.name, error)) + + if errors: + click.echo("ERRORS") + sys.exit(1) + + +@click.group() +def main(): + pass + + +@click.command() +def deploy(): + setup_riak() + + +@click.command() +def add_haproxies(): + setup_haproxies() + + +@click.command() +def undeploy(): + raise NotImplemented("Not yet") + + +main.add_command(deploy) +main.add_command(undeploy) +main.add_command(add_haproxies) + + +if __name__ == '__main__': + main() diff --git a/examples/riak/riaks.py b/examples/riak/riaks.py index 3ad0cbe6..e4a93e73 100644 --- a/examples/riak/riaks.py +++ b/examples/riak/riaks.py @@ -1,5 +1,10 @@ +#!/usr/bin/env python + # To run: # python example-riaks.py deploy +# solar changes stage +# solar changes process +# solar orch run-once last # python example-riaks.py add_haproxies # solar changes stage # solar changes process @@ -45,7 +50,7 @@ def setup_riak(): signals.connect(nodes[i], riak) for i, riak in enumerate(riak_services[1:]): - signals.connect(riak_services[0], riak, {'riak_name': 'join_to'}, events=None) + signals.connect(riak_services[0], riak, {'riak_name': 'join_to'}) hosts_services = [] for i, riak in enumerate(riak_services): @@ -62,21 +67,13 @@ def setup_riak(): 'ip': 'hosts:ip'}, events=False) + errors = resource.validate_resources() + for r, error in errors: + click.echo('ERROR: %s: %s' % (r.name, error)) has_errors = False - for r in locals().values(): - # TODO: handle list - if not isinstance(r, resource.Resource): - continue - - # print 'Validating {}'.format(r.name) - local_errors = validation.validate_resource(r) - if local_errors: - has_errors = True - print 'ERROR: %s: %s' % (r.name, local_errors) - - if has_errors: - print "ERRORS" + if errors: + click.echo("ERRORS") sys.exit(1) events = [ @@ -109,7 +106,7 @@ def setup_riak(): for event in events: add_event(event) - print 'Use solar changes process & orch' + click.echo('Use solar changes process & orch') sys.exit(0) @@ -219,6 +216,7 @@ def main(): def deploy(): setup_riak() + @click.command() def add_haproxies(): setup_haproxies() diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 3d670450..d057996d 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -47,7 +47,7 @@ def guess_mapping(emitter, receiver): return guessed -def connect(emitter, receiver, mapping={}, events=None): +def connect(emitter, receiver, mapping=None, events=None): mapping = mapping or guess_mapping(emitter, receiver) if isinstance(mapping, set): diff --git a/solar/solar/template.py b/solar/solar/template.py index c56546f5..7ce42e01 100644 --- a/solar/solar/template.py +++ b/solar/solar/template.py @@ -90,22 +90,23 @@ class ResourceTemplate(BaseTemplate): ) ) - def connect_list(self, resources, args={}): + def connect_list(self, resources, mapping=None): """Connect this resource to a ResourceListTemplate object. - args - optional connect mapping. This mapping can have the + mapping - optional connect mapping. This mapping can have the "{receiver_num}" string which enumerates each resrouce in resources list. """ + mapping = mapping or {} for receiver_num, resource in enumerate(resources.resources): kwargs = { 'receiver_num': receiver_num, } - args_fmt = self.args_fmt(args, kwargs) + mapping_fmt = self.args_fmt(mapping, kwargs) - signals.connect(self.resource, resource, args_fmt) + signals.connect(self.resource, resource, mapping_fmt) class ResourceListTemplate(BaseTemplate): @@ -117,15 +118,22 @@ class ResourceListTemplate(BaseTemplate): self.resources = resources @classmethod - def create(cls, count, resource_path, args={}): + def create(cls, + count, + resource_path, + name='{resource_path_name}-{num}', + args=None): """Create a number of resources of the same type, with optional args. + name -- optional resource name args -- an optional dict with create arguments. You can use "{num}" -- index of resource in the list "{resource_path_name}" -- name of resource from the `resource_path` argument """ + args = args or {} + created_resources = [] resource_path_name = os.path.split(resource_path)[-1] @@ -135,11 +143,11 @@ class ResourceListTemplate(BaseTemplate): 'num': num, 'resource_path_name': resource_path_name, } - kwargs['name'] = '{resource_path_name}-{num}'.format(**kwargs) + kwargs['name'] = name.format(**kwargs) args_fmt = cls.args_fmt(args, kwargs) - r = vr.create('{name}'.format(**kwargs), + r = vr.create(kwargs['name'], resource_path, args_fmt)[0] @@ -200,15 +208,16 @@ class ResourceListTemplate(BaseTemplate): return ResourceListTemplate(resources) - def connect_list(self, resources, args={}, events=None): + def connect_list(self, resources, mapping=None, events=None): """Connect self.resources to given resources in a 1-1 fashion. First resource in self.resources is connected to first resource in resources, second to second, etc. - args -- optional list of arguments + mapping -- optional mapping "{num}" -- substitutes for resource's index in args """ + mapping = mapping or {} for num, er in enumerate(zip(self.resources, resources.resources)): emitter, receiver = er @@ -217,19 +226,22 @@ class ResourceListTemplate(BaseTemplate): 'num': num, } - args_fmt = self.args_fmt(args, kwargs) + mapping_fmt = self.args_fmt(mapping, kwargs) - signals.connect(emitter, receiver, mapping=args_fmt, events=events) + signals.connect( + emitter, receiver, mapping=mapping_fmt, events=events + ) - def connect_list_to_each(self, resources, args={}, events=None): + def connect_list_to_each(self, resources, mapping=None, events=None): """Connect each resource in self.resources to each resource in resources. - args -- optional list of arguments - "{emitter_num}" -- substitutes for emitter's index in args (from + mapping -- optional mapping + "{emitter_num}" -- substitutes for emitter's index in mapping (from self.resources) - "{receiver_num}" -- substitutes for receiver's index in args (from + "{receiver_num}" -- substitutes for receiver's index in mapping (from resources argument) """ + mapping = mapping or {} for emitter_num, emitter in enumerate(self.resources): for receiver_num, receiver in enumerate(resources.resources): @@ -238,23 +250,24 @@ class ResourceListTemplate(BaseTemplate): 'receiver_num': receiver_num, } - args_fmt = self.args_fmt(args, kwargs) + mapping_fmt = self.args_fmt(mapping, kwargs) signals.connect( emitter, receiver, - mapping=args_fmt, + mapping=mapping_fmt, events=events ) - def on_each(self, resource_path, args={}): + def on_each(self, resource_path, args=None): """Create resource form resource_path on each resource in self.resources. """ + args = args or {} created_resources = ResourceListTemplate.create( len(self.resources), resource_path, - args + args=args ) for i, resource in enumerate(self.resources):