Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -36,3 +36,6 @@ vagrant-settings.yaml
 .solar_cli_uids
 
 .ssh/
+.cache
+
+.tox

examples/riak/riak_cluster.yaml

@@ -0,0 +1,149 @@
+id: riak_node
+
+resources:
+  - id: riak_service0
+    from: resources/riak_node
+    values:
+      riak_self_name: 'riak0'
+      riak_hostname: 'riak_server0.solar'
+      riak_name: 'riak0@riak_server0.solar'
+      ip: '{{nodes[0]}}::ip'
+      ssh_user: '{{nodes[0]}}::ssh_user'
+      ssh_key: '{{nodes[0]}}::ssh_key'
+
+{% for i in range(1, idx|int) %}
+  - id: riak_service{{i}}
+    from: resources/riak_node
+    values:
+      riak_self_name: 'riak{{i}}'
+      riak_hostname: 'riak_server{{i}}.solar'
+      riak_name: 'riak{{i}}@riak_server{{i}}.solar'
+      join_to: 'riak_service0::riak_name'
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: hosts_file{{i}}
+    from: resources/hosts_file
+    values:
+      hosts:name: 
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname::NO_EVENTS
+        {% endfor %}
+      hosts:ip:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::ip::NO_EVENTS
+        {% endfor %}
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service_config_http{{i}}
+    from: resources/haproxy_service_config
+    values:
+      listen_port: 8098
+      protocol: 'http'
+      name: 'riak_haproxy_http{{i}}'
+      backends:server:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_port_http
+        {% endfor %}
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service_config_pb{{i}}
+    from: resources/haproxy_service_config
+    values:
+      listen_port: 8087
+      protocol: 'tcp'
+      name: 'riak_haproxy_pb{{i}}'
+      backends:server:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_hostname
+        {% endfor %}
+      backends:port:
+        {% for j in range(idx|int) %}
+        - riak_service{{j}}::riak_port_pb
+        {% endfor %}
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_config{{i}}
+    from: resources/haproxy_config
+    values:
+      config:protocol:
+        - haproxy_service_config_http{{i}}::protocol
+        - haproxy_service_config_pb{{i}}::protocol
+      config:listen_port:
+        - haproxy_service_config_http{{i}}::listen_port
+        - haproxy_service_config_pb{{i}}::listen_port
+      config:name:
+        - haproxy_service_config_http{{i}}::name
+        - haproxy_service_config_pb{{i}}::name
+      config:backends:
+        - haproxy_service_config_http{{i}}::backends
+        - haproxy_service_config_pb{{i}}::backends
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+{% for i in range(idx|int) %}
+  - id: haproxy_service{{i}}
+    from: resources/haproxy_service
+    values:
+      ip: '{{nodes[i]}}::ip'
+      ssh_user: '{{nodes[i]}}::ssh_user'
+      ssh_key: '{{nodes[i]}}::ssh_key'
+{% endfor %}
+
+
+events:
+{% for i in range(idx|int) %}
+  - type: depends_on
+    parent_action: 'hosts_file{{i}}.run'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.run'
+{% endfor %}
+
+{% for i in range(1, idx|int) %}
+  - type: react_on
+    parent_action: 'riak_service{{i}}.run'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.join'
+
+  - type: react_on
+    parent_action: 'riak_service{{i}}.leave'
+    state: 'success'
+    depend_action: 'riak_service{{i}}.join'
+
+  - type: react_on
+    parent_action: 'riak_service{{i}}.join'
+    state: 'success'
+    depend_action: 'riak_service0.commit'
+{% endfor %}
+
+{% for i in range(1, idx|int) %}
+  - type: depends_on
+    parent_action: 'haproxy_service{{i}}.run'
+    state: 'success'
+    depend_action: 'haproxy_config{{i}}.run'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{i}}.run'
+    state: 'success'
+    depend_action: 'haproxy_service{{i}}.apply_config'
+
+  - type: react_on
+    parent_action: 'haproxy_config{{i}}.update'
+    state: 'success'
+    depend_action: 'haproxy_service{{i}}.apply_config'
+{% endfor %}

examples/riak/riaks-template.py

@@ -4,116 +4,174 @@
 # work here, for most up-to-date version see example-riaks.py
 # This is just a demo of the template language of Solar
 
+import click
+import sys
+
+from solar.core import resource
 from solar.interfaces.db import get_db
-from solar.core import signals
-
-db = get_db()
-db.clear()
-signals.Connections.clear()
-
 from solar import template
 
 
-nodes = template.nodes_from('templates/riak_nodes.yaml')
-
-riak_services = nodes.on_each(
-    'resources/riak_node',
-    {
-        'riak_self_name': 'riak{num}',
-        'riak_hostname': 'riak_server{num}.solar',
-        'riak_name': 'riak{num}@riak_server{num}.solar',
-})
-
-slave_riak_services = riak_services.tail()
-
-riak_services.take(0).connect_list(
-    slave_riak_services,
-    {
-        'riak_name': 'join_to',
-    }
-)
-
-hosts_files = nodes.on_each('resources/hosts_file')
-
-riak_services.connect_list_to_each(
-    hosts_files,
-    {
-        'ip': 'hosts_ips',
-        'riak_hostname': 'hosts_names',
-    },
-    events=False
-)
+db = get_db()
 
 
-hosts_files.add_deps('run/success', riak_services, 'run')
-slave_riak_services.add_reacts('run/success', slave_riak_services, 'join')
-slave_riak_services.add_reacts('leave/success', slave_riak_services, 'join')
-slave_riak_services.add_react('run/success', riak_services.take(0), 'commit')
+def setup_riak():
+    db.clear()
+
+    nodes = template.nodes_from('templates/riak_nodes.yaml')
+
+    riak_services = nodes.on_each(
+        'resources/riak_node',
+        args={
+            'riak_self_name': 'riak{num}',
+            'riak_hostname': 'riak_server{num}.solar',
+            'riak_name': 'riak{num}@riak_server{num}.solar',
+        }
+    )
+
+    slave_riak_services = riak_services.tail()
+
+    riak_services.take(0).connect_list(
+        slave_riak_services,
+        mapping={
+            'riak_name': 'join_to',
+        }
+    )
+
+    hosts_files = nodes.on_each('resources/hosts_file')
+
+    riak_services.connect_list_to_each(
+        hosts_files,
+        mapping={
+            'ip': 'hosts:ip',
+            'riak_hostname': 'hosts:name',
+        },
+        events=False
+    )
+
+    errors = resource.validate_resources()
+    for r, error in errors:
+        click.echo('ERROR: %s: %s' % (r.name, error))
+
+    if errors:
+        click.echo("ERRORS")
+        sys.exit(1)
+
+    hosts_files.add_deps('run/success', riak_services, 'run')
+    slave_riak_services.add_reacts('run/success', slave_riak_services, 'join')
+    slave_riak_services.add_reacts('leave/success', slave_riak_services, 'join')
+    slave_riak_services.add_react('run/success', riak_services.take(0), 'commit')
 
 
-haproxy_services = nodes.on_each(
-    'resources/haproxy_service'
-)
-haproxy_configs = nodes.on_each(
-    'resources/haproxy_config'
-)
-haproxy_service_configs_http = nodes.on_each(
-    'resources/haproxy_service_config',
-    {
-        'listen_port': 8098,
-        'protocol': 'http',
-    }
-)
-haproxy_service_configs_pb = nodes.on_each(
-    'resources/haproxy_service_config',
-    {
-        'listen_port': 8087,
-        'protocol': 'tcp',
-    }
-)
+def setup_haproxies():
+    # TODO: VR loading needs to be supported, then we can do something like
+    # nodes = template.load('nodes')
 
-riak_services.connect_list_to_each(
-    haproxy_service_configs_http,
-    {
-        'riak_hostname': 'servers',
-        'riak_port_http': 'ports',
-    }
-)
-riak_services.connect_list_to_each(
-    haproxy_service_configs_pb,
-    {
-        'riak_hostname': 'servers',
-        'riak_port_pb': 'ports',
-    }
-)
-haproxy_service_configs_http.connect_list(
-    haproxy_configs,
-    {
-        'protocol': 'configs_protocols',
-        'listen_port': 'listen_ports',
-        'name': 'configs_names',
-        'servers': 'configs',
-        'ports': 'configs_ports',
-    }
-)
-haproxy_service_configs_pb.connect_list(
-    haproxy_configs,
-    {
-        'protocol': 'configs_protocols',
-        'listen_port': 'listen_ports',
-        'name': 'configs_names',
-        'servers': 'configs',
-        'ports': 'configs_ports',
-    }
-)
-haproxy_configs.connect_list(
-    haproxy_services,
-    {
-        'listen_ports': 'ports',
-    }
-)
+    nodes = template.ResourceListTemplate([
+        resource.load('node1'),
+        resource.load('node2'),
+        resource.load('node3'),
+    ])
+    riak_services = template.ResourceListTemplate([
+        resource.load('riak_node-0'),
+        resource.load('riak_node-1'),
+        resource.load('riak_node-2'),
+    ])
 
-nodes.add_reacts('run/success', haproxy_services, 'install')
-haproxy_services.add_deps('install/success', haproxy_configs, 'run')
-haproxy_configs.add_reacts('run/success', haproxy_services, 'run')
-haproxy_configs.add_reacts('update/success', haproxy_services, 'update')
+    haproxy_services = nodes.on_each(
+        'resources/haproxy_service'
+    )
+    haproxy_configs = nodes.on_each(
+        'resources/haproxy_config'
+    )
+    haproxy_service_configs_http = riak_services.on_each(
+        'resources/haproxy_service_config',
+        {
+            'listen_port': 8098,
+            'protocol': 'http',
+            'name': 'riak_haproxy_http{num}',
+        }
+    )
+    haproxy_service_configs_pb = riak_services.on_each(
+        'resources/haproxy_service_config',
+        {
+            'listen_port': 8087,
+            'protocol': 'tcp',
+            'name': 'riak_haproxy_pb{num}',
+        }
+    )
+
+    riak_services.connect_list_to_each(
+        haproxy_service_configs_http,
+        {
+            'riak_hostname': 'backends:server',
+            'riak_port_http': 'backends:port',
+        }
+    )
+    riak_services.connect_list_to_each(
+        haproxy_service_configs_pb,
+        {
+            'riak_hostname': 'backends:server',
+            'riak_port_pb': 'backends:port',
+        }
+    )
+    haproxy_service_configs_http.connect_list(
+        haproxy_configs,
+        {
+            'backends': 'config:backends',
+            'listen_port': 'config:listen_port',
+            'protocol': 'config:protocol',
+            'name': 'config:name',
+        }
+    )
+    haproxy_service_configs_pb.connect_list(
+        haproxy_configs,
+        {
+            'backends': 'config:backends',
+            'listen_port': 'config:listen_port',
+            'protocol': 'config:protocol',
+            'name': 'config:name',
+        }
+    )
+
+    #nodes.add_reacts('run/success', haproxy_services, 'install')
+    haproxy_services.add_deps('run/success', haproxy_configs, 'run')
+    haproxy_configs.add_reacts('run/success', haproxy_services, 'apply_config')
+    haproxy_configs.add_reacts('update/success', haproxy_services, 'apply_config')
+
+    errors = resource.validate_resources()
+    for r, error in errors:
+        click.echo('ERROR: %s: %s' % (r.name, error))
+
+    if errors:
+        click.echo("ERRORS")
+        sys.exit(1)
+
+
+@click.group()
+def main():
+    pass
+
+
+@click.command()
+def deploy():
+    setup_riak()
+
+
+@click.command()
+def add_haproxies():
+    setup_haproxies()
+
+
+@click.command()
+def undeploy():
+    raise NotImplemented("Not yet")
+
+
+main.add_command(deploy)
+main.add_command(undeploy)
+main.add_command(add_haproxies)
+
+
+if __name__ == '__main__':
+    main()

examples/riak/riaks.py

@@ -1,5 +1,10 @@
+#!/usr/bin/env python
+
 # To run:
 # python example-riaks.py deploy
+# solar changes stage
+# solar changes process
+# solar orch run-once last
 # python example-riaks.py add_haproxies
 # solar changes stage
 # solar changes process
@@ -45,7 +50,7 @@ def setup_riak():
         signals.connect(nodes[i], riak)
 
     for i, riak in enumerate(riak_services[1:]):
-        signals.connect(riak_services[0], riak, {'riak_name': 'join_to'}, events=None)
+        signals.connect(riak_services[0], riak, {'riak_name': 'join_to'})
 
     hosts_services = []
     for i, riak in enumerate(riak_services):
@@ -58,24 +63,17 @@ def setup_riak():
     for riak in riak_services:
         for hosts_file in hosts_services:
             signals.connect(riak, hosts_file,
-                            {'riak_hostname': 'hosts_names', 'ip': 'hosts_ips'},
+                            {'riak_hostname': 'hosts:name',
+                             'ip': 'hosts:ip'},
                             events=False)
 
+    errors = resource.validate_resources()
+    for r, error in errors:
+        click.echo('ERROR: %s: %s' % (r.name, error))
     has_errors = False
-    for r in locals().values():
 
-        # TODO: handle list
-        if not isinstance(r, resource.Resource):
-            continue
-
-        # print 'Validating {}'.format(r.name)
-        local_errors = validation.validate_resource(r)
-        if local_errors:
-            has_errors = True
-            print 'ERROR: %s: %s' % (r.name, local_errors)
-
-    if has_errors:
-        print "ERRORS"
+    if errors:
+        click.echo("ERRORS")
         sys.exit(1)
 
     events = [
@@ -108,7 +106,7 @@ def setup_riak():
     for event in events:
         add_event(event)
 
-    print 'Use solar changes process & orch'
+    click.echo('Use solar changes process & orch')
     sys.exit(0)
 
 
@@ -143,33 +141,28 @@ def setup_haproxies():
 
     for single_hpsc in hpsc_http:
         for riak in riaks:
-            signals.connect(riak, single_hpsc, {'riak_hostname': 'servers',
-                                                'riak_port_http': 'ports'})
+            signals.connect(riak, single_hpsc, {'riak_hostname': 'backends:server',
+                                                'riak_port_http': 'backends:port'})
 
     for single_hpsc in hpsc_pb:
         for riak in riaks:
-            signals.connect(riak, single_hpsc, {'riak_hostname': 'servers',
-                                                'riak_port_pb': 'ports'})
+            signals.connect(riak, single_hpsc, {'riak_hostname': 'backends:server',
+                                                'riak_port_pb': 'backends:port'})
 
     # haproxy config to haproxy service
 
     for single_hpc, single_hpsc in zip(hpc, hpsc_http):
-        signals.connect(single_hpsc, single_hpc, {'protocol': 'configs_protocols',
-                                                  'listen_port': 'listen_ports',
-                                                  'name': 'configs_names',
-                                                  'servers': 'configs',
-                                                  'ports': 'configs_ports'})
+        signals.connect(single_hpsc, single_hpc, {"backends": "config:backends",
+                                                  "listen_port": "config:listen_port",
+                                                  "protocol": "config:protocol",
+                                                  "name": "config:name"})
 
     for single_hpc, single_hpsc in zip(hpc, hpsc_pb):
-        signals.connect(single_hpsc, single_hpc, {'protocol': 'configs_protocols',
-                                                  'listen_port': 'listen_ports',
-                                                  'name': 'configs_names',
-                                                  'servers': 'configs',
-                                                  'ports': 'configs_ports'})
+        signals.connect(single_hpsc, single_hpc, {"backends": "config:backends",
+                                                  "listen_port": "config:listen_port",
+                                                  "protocol": "config:protocol",
+                                                  "name": "config:name"})
 
-    for single_hps, single_hpc in zip(hps, hpc):
-        signals.connect(single_hpc, single_hps, {'listen_ports': 'ports'},
-                        events=False)
 
     # assign haproxy services to each node
 
@@ -223,6 +216,7 @@ def main():
 def deploy():
     setup_riak()
 
+
 @click.command()
 def add_haproxies():
     setup_haproxies()

resources/haproxy_config/actions/run.yaml

@@ -2,21 +2,21 @@
 - hosts: [{{host}}]
   sudo: yes
   vars:
-    config_dir: {src: {{ config_dir.value['src'] }}, dst: {{ config_dir.value['dst'] }}}
+    config_dir: {src: {{ config_dir['src'] }}, dst: {{ config_dir['dst'] }}}
     haproxy_ip: {{ ip }}
     haproxy_services:
-      {% for service, ports, listen_port, protocol in zip(configs.value, configs_ports.value, listen_ports.value, configs_protocols.value) %}
-      - name: {{ service['emitter_attached_to'] }}
-        listen_port: {{ listen_port['value'] }}
-        protocol: {{ protocol['value'] }}
-        servers:
-          {% for server_ip, server_port in zip(service['value'], ports['value']) %}
-          - name: {{ server_ip['emitter_attached_to'] }}
-            ip: {{ server_ip['value'] }}
-            port: {{ server_port['value'] }}
-          {% endfor %}
+      {% for single in config %}
+        - name: {{ single['name'] }}
+          listen_port: {{ single['listen_port'] }}
+          protocol: {{ single['protocol'] }}
+          servers:
+            {% for backend in single['backends'] %}
+            - name: {{ backend['server'] }}_{{ backend['port'] }}
+              ip: {{ backend['server'] }}
+              port: {{ backend['port'] }}
+            {% endfor %}
       {% endfor %}
   tasks:
-    - file: path={{ config_dir.value['src'] }}/ state=directory
-    - file: path={{ config_dir.value['src'] }}/haproxy.cfg state=touch
+    - file: path={{ config_dir['src'] }}/ state=directory
+    - file: path={{ config_dir['src'] }}/haproxy.cfg state=touch
     - template: src={{templates_dir}}/haproxy.cfg dest=/etc/haproxy/haproxy.cfg

resources/haproxy_config/actions/update.yaml

@@ -1,22 +1,22 @@
 # TODO
-- hosts: [{{ip}}]
+- hosts: [{{host}}]
   sudo: yes
   vars:
-    config_dir: {src: {{ config_dir.value['src'] }}, dst: {{ config_dir.value['dst'] }}}
+    config_dir: {src: {{ config_dir['src'] }}, dst: {{ config_dir['dst'] }}}
     haproxy_ip: {{ ip }}
     haproxy_services:
-      {% for service, ports, listen_port, protocol in zip(configs.value, configs_ports.value, listen_ports.value, configs_protocols.value) %}
-      - name: {{ service['emitter_attached_to'] }}
-        listen_port: {{ listen_port['value'] }}
-        protocol: {{ protocol['value'] }}
-        servers:
-          {% for server_ip, server_port in zip(service['value'], ports['value']) %}
-          - name: {{ server_ip['emitter_attached_to'] }}
-            ip: {{ server_ip['value'] }}
-            port: {{ server_port['value'] }}
-          {% endfor %}
+      {% for single in config %}
+        - name: {{ single['name'] }}
+          listen_port: {{ single['listen_port'] }}
+          protocol: {{ single['protocol'] }}
+          servers:
+            {% for backend in single['backends'] %}
+            - name: {{ backend['server'] }}_{{ backend['port'] }}
+              ip: {{ backend['server'] }}
+              port: {{ backend['port'] }}
+            {% endfor %}
       {% endfor %}
   tasks:
-    - file: path={{ config_dir.value['src'] }}/ state=directory
-    - file: path={{ config_dir.value['src'] }}/haproxy.cfg state=touch
+    - file: path={{ config_dir['src'] }}/ state=directory
+    - file: path={{ config_dir['src'] }}/haproxy.cfg state=touch
     - template: src={{templates_dir}}/haproxy.cfg dest=/etc/haproxy/haproxy.cfg

resources/haproxy_config/meta.yaml

@@ -8,20 +8,8 @@ input:
     config_dir:
       schema: {src: str!, dst: str!}
       value: {src: /etc/solar/haproxy, dst: /etc/haproxy}
-    listen_ports:
-      schema: [int]
-      value: []
-    configs:
-      schema: [[str]]
-      value: []
-    configs_names:
-      schema: [str]
-      value: []
-    configs_ports:
-      schema: [[int]]
-      value: []
-    configs_protocols:
-      schema: [str]
+    config:
+      schema: [{backends: [{server: str!, port: int!}], listen_port: int!, protocol: str!, name: str!}]
       value: []
     ssh_user:
       schema: str!

resources/haproxy_service/meta.yaml

@@ -5,9 +5,6 @@ input:
     ip:
       schema: str!
       value:
-    ports:
-      schema: [[int]]
-      value: []
     ssh_user:
       schema: str!
       value:

resources/haproxy_service_config/meta.yaml

@@ -5,17 +5,19 @@ input:
     name:
       schema: str!
       value: general_haproxy
+    backends:
+      schema: [{server: str!, port: int!}]
     listen_port:
       schema: int!
       value: 9999
     protocol:
       schema: str!
       value: http
-    ports:
-      schema: [int]
-      value: []
-    servers:
-      schema: [str]
-      value: []
+    # ports:
+    #   schema: [int]
+    #   value: []
+    # servers:
+    #   schema: [str]
+    #   value: []
 
 tags: [resources/haproxy, resource/haproxy_general_config]

resources/ro_node/meta.yaml

@@ -6,9 +6,6 @@ input:
     ip:
       schema: str!
       value:
-    name:
-      schema: str
-      value:
     ssh_key:
       schema: str!
       value:
@@ -17,4 +14,4 @@ input:
       value:
     name:
       schema: str
-      value: a node
+      value:

templates/riak_nodes.yaml

@@ -4,17 +4,20 @@ resources:
     from: resources/ro_node
     values:
       ip: '10.0.0.3'
+      name: 'node1'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key'
       ssh_user: 'vagrant'
   - id: node2
     from: resources/ro_node
     values:
       ip: '10.0.0.4'
+      name: 'node2'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev2/virtualbox/private_key'
       ssh_user: 'vagrant'
   - id: node3
     from: resources/ro_node
     values:
       ip: '10.0.0.5'
+      name: 'node3'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev3/virtualbox/private_key'
       ssh_user: 'vagrant'