Merge branch 'x-fixes' into x

Vagrantfile

@@ -29,6 +29,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.define "solar-dev2" do |guest2|
+    guest2.vm.provision "shell", inline: init_script, privileged: true
     guest2.vm.network "private_network", ip: "10.0.0.3"
     guest2.vm.host_name = "solar-dev2"
 
@@ -39,6 +40,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.define "solar-dev3" do |guest3|
+    guest3.vm.provision "shell", inline: init_script, privileged: true
     guest3.vm.network "private_network", ip: "10.0.0.4"
     guest3.vm.host_name = "solar-dev3"
 
@@ -49,6 +51,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.define "solar-dev4" do |guest4|
+    guest4.vm.provision "shell", inline: init_script, privileged: true
     guest4.vm.network "private_network", ip: "10.0.0.5"
     guest4.vm.host_name = "solar-dev4"
 
@@ -59,6 +62,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.define "solar-dev5" do |guest5|
+    guest5.vm.provision "shell", inline: init_script, privileged: true
     guest5.vm.network "private_network", ip: "10.0.0.6"
     guest5.vm.host_name = "solar-dev5"
 
@@ -69,8 +73,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.define "solar-dev6" do |guest6|
+    guest6.vm.provision "shell", inline: init_script, privileged: true
     guest6.vm.network "private_network", ip: "10.0.0.7"
-    guest6.vm.host_name = "solar-dev5"
+    guest6.vm.host_name = "solar-dev6"
 
     guest6.vm.provider :virtualbox do |v|
       v.customize ["modifyvm", :id, "--memory", 256]

cli.py

@@ -1,10 +1,11 @@
 import click
 import json
-import matplotlib
-matplotlib.use('Agg')  # don't show windows
-import matplotlib.pyplot as plt
+#import matplotlib
+#matplotlib.use('Agg')  # don't show windows
+#import matplotlib.pyplot as plt
 import networkx as nx
 import os
+import subprocess
 
 from x import actions as xa
 from x import deployment as xd
@@ -145,13 +146,19 @@ def init_cli_connections():
     # TODO: this requires graphing libraries
     @click.command()
     def graph():
-        g = xs.connection_graph()
-        pos = nx.spring_layout(g)
-        nx.draw_networkx_nodes(g, pos)
-        nx.draw_networkx_edges(g, pos, arrows=True)
-        nx.draw_networkx_labels(g, pos)
-        plt.axis('off')
-        plt.savefig('graph.png')
+        #g = xs.connection_graph()
+        g = xs.detailed_connection_graph()
+
+        nx.write_dot(g, 'graph.dot')
+        subprocess.call(['dot', '-Tpng', 'graph.dot', '-o', 'graph.png'])
+
+        # Matplotlib
+        #pos = nx.spring_layout(g)
+        #nx.draw_networkx_nodes(g, pos)
+        #nx.draw_networkx_edges(g, pos, arrows=True)
+        #nx.draw_networkx_labels(g, pos)
+        #plt.axis('off')
+        #plt.savefig('graph.png')
 
     connections.add_command(graph)
 

haproxy.cfg

@@ -0,0 +1,46 @@
+global
+  log 127.0.0.1 local0
+  log 127.0.0.1 local1 notice
+  maxconn 4096
+  tune.ssl.default-dh-param 2048
+  pidfile /var/run/haproxy.pid
+  user haproxy
+  group haproxy
+  daemon
+  stats socket /var/run/haproxy.stats level admin
+  ssl-default-bind-options no-sslv3
+
+defaults
+  log global
+  mode http
+  option redispatch
+  option httplog
+  option dontlognull
+  option forwardfor
+  timeout connect 5000
+  timeout client 50000
+  timeout server 50000
+
+#frontend default_frontend
+#  bind 0.0.0.0:80
+#  default_backend default_service
+
+#backend default_service
+#  balance roundrobin
+
+{% for service in haproxy_services %}
+listen {{ service['name'] }} 0.0.0.0:{{ service['listen_port'] }}
+    mode http
+    stats enable
+    stats uri /haproxy?stats
+    stats realm Strictly\ Private
+    stats auth A_Username:YourPassword
+    stats auth Another_User:passwd
+    balance roundrobin
+    option httpclose
+    option forwardfor
+    {% for server in service['servers'] %}
+    server {{ server['name'] }} {{ server['ip'] }}:{{ server['port'] }} check
+    {% endfor %}
+
+{% endfor %}

haproxy_deployment/haproxy-deployment.yaml

@@ -72,6 +72,8 @@ resources:
     model: x/resources/haproxy_config/
     args:
       servers: {}
+      listen_port: 5000
+      ports: {}
       ssh_user:
       ssh_key:
 
@@ -111,33 +113,30 @@ resources:
     model: x/resources/haproxy_config/
     args:
       servers: {}
+      listen_port: 8774
+      ports: {}
       ssh_user:
       ssh_key:
 
-  #- name: haproxy-config-container
-  #  model: x/resources/data_container/
-  #  args:
-  #    ip:
-  #    image: haproxy-config
-  #    export_volumes:
-  #      - haproxy-config
-  #- name: haproxy-config
-  #  model: x/resources/haproxy/
-  #  args:
-  #    ip:
-  #    configs: {}
-  #    ssh_user:
-  #    ssh_key:
+  - name: haproxy-config
+    model: x/resources/haproxy/
+    args:
+      ip:
+      listen_ports: {}
+      configs: {}
+      configs_ports: {}
+      ssh_user:
+      ssh_key:
   - name: haproxy
     model: x/resources/docker_container
     args:
       ip:
-      image: haproxy
+      image: tutum/haproxy
+      ports: {}
       ssh_user:
       ssh_key:
-      host_binds:
-        - /etc/haproxy: /vagrant/haproxy-etc
-      volume_binds:
+      host_binds: {}
+      volume_binds: {}
 
 
 connections:
@@ -153,10 +152,12 @@ connections:
     receiver: haproxy_keystone_config
     mapping:
       ip: servers
+      port: ports
   - emitter: keystone2
     receiver: haproxy_keystone_config
     mapping:
       ip: servers
+      port: ports
 
   - emitter: node3
     receiver: mariadb_nova1_data
@@ -170,30 +171,35 @@ connections:
     receiver: haproxy_nova_config
     mapping:
       ip: servers
+      port: ports
   - emitter: nova2
     receiver: haproxy_nova_config
     mapping:
       ip: servers
+      port: ports
 
   # HAProxy config container
-  #- emitter: node5
-  #  receiver: haproxy-config-container
-  #- emitter: haproxy-config-container
-  #  receiver: haproxy-config
-  #- emitter: haproxy_keystone_config
-  #  receiver: haproxy-config
-  #  mapping:
-  #    servers: configs
-  #- emitter: haproxy_nova_config
-  #  receiver: haproxy-config
-  #  mapping:
-  #    servers: configs
-
-  # HAProxy service
   - emitter: node5
+    receiver: haproxy-config
+
+  - emitter: haproxy_keystone_config
+    receiver: haproxy-config
+    mapping:
+      listen_port: listen_ports
+      ports: configs_ports
+      servers: configs
+  - emitter: haproxy_nova_config
+    receiver: haproxy-config
+    mapping:
+      listen_port: listen_ports
+      ports: configs_ports
+      servers: configs
+
+  - emitter: haproxy-config
     receiver: haproxy
-
-  #- emitter: haproxy-config
-  #  receiver: haproxy
-  #  mapping:
-
+    mapping:
+      ip: ip
+      listen_ports: ports
+      ssh_user: ssh_user
+      ssh_key: ssh_key
+      config_dir: host_binds

haproxy_deployment/haproxy_deployment.py

@@ -25,6 +25,13 @@ class TestHAProxyDeployment(unittest.TestCase):
                 'keystone2': keystone2.args['ip'],
             }
         )
+        self.assertDictEqual(
+            haproxy_keystone_config.args['ports'],
+            {
+                'keystone1': keystone1.args['port'],
+                'keystone2': keystone2.args['port'],
+            }
+        )
 
     def test_nova_config(self):
         node3 = db.get_resource('node3')
@@ -47,21 +54,55 @@ class TestHAProxyDeployment(unittest.TestCase):
                 'nova2': nova2.args['ip'],
             }
         )
+        self.assertDictEqual(
+            haproxy_nova_config.args['ports'],
+            {
+                'nova1': nova1.args['port'],
+                'nova2': nova2.args['port'],
+            }
+        )
 
     def test_haproxy(self):
         node5 = db.get_resource('node5')
         haproxy_keystone_config = db.get_resource('haproxy_keystone_config')
         haproxy_nova_config = db.get_resource('haproxy_nova_config')
         haproxy = db.get_resource('haproxy')
+        haproxy_config = db.get_resource('haproxy-config')
 
         self.assertEqual(node5.args['ip'], haproxy.args['ip'])
-        #self.assertItemsEqual(
-        #    haproxy.args['configs'],
-        #    {
-        #        'haproxy_keystone_config': haproxy_keystone_config.args['servers'],
-        #        'haproxy_nova_config': haproxy_nova_config.args['servers'],
-        #    }
-        #)
+        self.assertEqual(node5.args['ssh_key'], haproxy.args['ssh_key'])
+        self.assertEqual(node5.args['ssh_user'], haproxy.args['ssh_user'])
+        self.assertDictEqual(
+            haproxy_config.args['configs'],
+            {
+                'haproxy_keystone_config': haproxy_keystone_config.args['servers'],
+                'haproxy_nova_config': haproxy_nova_config.args['servers'],
+            }
+        )
+        self.assertDictEqual(
+            haproxy_config.args['configs_ports'],
+            {
+                'haproxy_keystone_config': haproxy_keystone_config.args['ports'],
+                'haproxy_nova_config': haproxy_nova_config.args['ports'],
+            }
+        )
+        self.assertDictEqual(
+            haproxy_config.args['listen_ports'],
+            {
+                'haproxy_keystone_config': haproxy_keystone_config.args['listen_port'],
+                'haproxy_nova_config': haproxy_nova_config.args['listen_port'],
+            }
+        )
+        self.assertDictEqual(
+            {
+                'haproxy-config': haproxy_config.args['config_dir'],
+            },
+            haproxy.args['host_binds']
+        )
+        self.assertDictEqual(
+            haproxy.args['ports'],
+            haproxy_config.args['listen_ports'],
+        )
 
 
 def main():

main.yml

@@ -16,7 +16,8 @@
     - shell: pip install -r /vagrant/requirements.txt
 
     # Graph drawing
-    - apt: name=python-matplotlib state=present
+    #- apt: name=python-matplotlib state=present
+    - apt: name=python-graphviz state=present
 
       # Setup development env for solar
     #- shell: python setup.py develop chdir=/vagrant/solar

x/db.py

@@ -11,3 +11,9 @@ def resource_add(key, value):
 
 def get_resource(key):
     return RESOURCE_DB.get(key, None)
+
+
+def clear():
+    global RESOURCE_DB
+
+    RESOURCE_DB = {}

x/deployment.py

@@ -17,9 +17,8 @@ def deploy(filename):
     resource_save_path = os.path.join(workdir, config['resource-save-path'])
 
     # Clean stuff first
-    clients_file = os.path.join(workdir, 'clients.json')
-    if os.path.exists(clients_file):
-        os.remove(clients_file)
+    db.clear()
+    xs.clear()
     shutil.rmtree(resource_save_path, ignore_errors=True)
     os.makedirs(resource_save_path)
 

x/handlers/ansible.py

@@ -1,6 +1,7 @@
 # -*- coding: UTF-8 -*-
 import os
 import subprocess
+import yaml
 
 from x.handlers.base import BaseHandler
 
@@ -11,7 +12,9 @@ class Ansible(BaseHandler):
         playbook_file = self._create_playbook(resource, action_name)
         print 'inventory_file', inventory_file
         print 'playbook_file', playbook_file
-        subprocess.call(['ansible-playbook', '-i', inventory_file, playbook_file])
+        call_args = ['ansible-playbook', '-i', inventory_file, playbook_file]
+        print 'EXECUTING: ', ' '.join(call_args)
+        subprocess.call(call_args)
 
     #def _get_connection(self, resource):
     #    return {'ssh_user': '',

x/resource.py

@@ -19,7 +19,7 @@ class Resource(object):
         self.metadata = metadata
         self.actions = metadata['actions'].keys() if metadata['actions'] else None
         self.requires = metadata['input'].keys()
-        self._validate_args(args)
+        self._validate_args(args, metadata['input'])
         self.args = args
         self.metadata['input'] = args
         self.input_types = metadata.get('input-types', {})
@@ -63,10 +63,15 @@ class Resource(object):
         else:
             raise Exception('Uuups, action is not available')
 
-    def _validate_args(self, args):
+    def _validate_args(self, args, inputs):
         for req in self.requires:
             if req not in args:
-                raise Exception('Requirement `{0}` is missing in args'.format(req))
+                # If metadata input is filled with a value, use it as default
+                # and don't report an error
+                if inputs.get(req):
+                    args[req] = inputs[req]
+                else:
+                    raise Exception('Requirement `{0}` is missing in args'.format(req))
 
     # TODO: versioning
     def save(self):

x/resources/docker_container/actions/run.yml

@@ -2,5 +2,23 @@
 - hosts: [{{ ip }}]
   sudo: yes
   tasks:
-    - shell: docker run -d --net="host" --privileged \
-        --name {{ name }} {{ image }}
+    - apt: name=python-pip state=present
+    - shell: pip install docker-py
+    - service: name=docker state=started
+    - docker:
+       name: {{ name }}
+       image: {{ image }}
+       state: running
+       ports:
+         {% for name, port in ports.items() %}
+         # {{ name }}
+         - {{ port }}:{{ port }}
+         {% endfor %}
+       volumes:
+         # TODO: host_binds might need more work
+         # Currently it's not that trivial to pass custom src: dst here
+         # (when a config variable is passed here from other resource)
+         # so we mount it to the same directory as on host
+         {% for emitter, bind in host_binds.items() %}
+         - {{ bind }}:{{ bind }}
+         {% endfor %}

x/resources/docker_container/meta.yaml

@@ -4,8 +4,12 @@ version: 1.0.0
 input:
     ip:
     image:
+    ports:
     host_binds:
     volume_binds:
+    ssh_user:
+    ssh_key:
 input-types:
+    ports:
     host_binds: list
     volume_binds: list

x/resources/haproxy/actions/run.yml

@@ -1,6 +1,23 @@
 # TODO
 - hosts: [{{ ip }}]
   sudo: yes
+  vars:
+    config_dir: {{ config_dir }}
+    haproxy_ip: {{ ip }}
+    haproxy_services:
+      {% for service, servers in configs.items() %}
+      - name: {{ service }}
+        listen_port: {{ listen_ports[service] }}
+        servers:
+          {% for name, ip in servers.items() %}
+          - name: {{ name }}
+            ip: {{ ip }}
+            port: {{ configs_ports[service][name] }}
+          {% endfor %}
+      {% endfor %}
   tasks:
-    - shell: docker run -d --net="host" --privileged \
-        --name {{ name }} {{ image }}
+    - apt: name=python-pip state=present
+    - shell: pip install docker-py
+    - service: name=docker state=started
+    - file: path=/etc/haproxy/haproxy.cfg state=touch
+    - template: src=/vagrant/haproxy.cfg dest=/etc/haproxy/haproxy.cfg

x/resources/haproxy/meta.yaml

@@ -3,6 +3,13 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    config_dir: /etc/haproxy
+    listen_ports:
     configs:
+    configs_ports:
+    ssh_user:
+    ssh_key:
 input-types:
+    listen_ports: list
     configs: list
+    configs_ports: list

x/resources/haproxy_config/meta.yaml

@@ -2,6 +2,9 @@ id: haproxy_config
 handler: ansible
 version: 1.0.0
 input:
+    listen_port:
+    ports:
     servers:
 input-types:
+    ports: list
     servers: list

x/resources/keystone/meta.yaml

@@ -3,4 +3,5 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    port: 5000
     image: garland/docker-openstack-keystone

x/resources/nova/meta.yaml

@@ -3,4 +3,5 @@ handler: ansible
 version: 1.0.0
 input:
     ip:
+    port: 8774
     image: # TODO

x/signals.py

@@ -2,6 +2,7 @@
 from collections import defaultdict
 import itertools
 import networkx as nx
+import os
 
 import db
 
@@ -12,6 +13,16 @@ CLIENTS_CONFIG_KEY = 'clients-data-file'
 CLIENTS = utils.read_config_file(CLIENTS_CONFIG_KEY)
 
 
+def clear():
+    global CLIENTS
+
+    CLIENTS = {}
+
+    path = utils.read_config()[CLIENTS_CONFIG_KEY]
+    if os.path.exists(path):
+        os.remove(path)
+
+
 def guess_mapping(emitter, receiver):
     """Guess connection mapping between emitter and receiver.
 
@@ -155,3 +166,17 @@ def connection_graph():
         )
 
     return g
+
+
+def detailed_connection_graph():
+    g = nx.MultiDiGraph()
+
+    for emitter_name, destination_values in CLIENTS.items():
+        for emitter_input, receivers in CLIENTS[emitter_name].items():
+            for receiver_name, receiver_input in receivers:
+                label = emitter_input
+                if emitter_input != receiver_input:
+                    label = '{}:{}'.format(emitter_input, receiver_input)
+                g.add_edge(emitter_name, receiver_name, label=label)
+
+    return g

x/test/__init__.py

@@ -0,0 +1 @@
+__author__ = 'przemek'

x/test/base.py

@@ -0,0 +1,36 @@
+import os
+import shutil
+import tempfile
+import unittest
+import yaml
+
+from x import db
+from x import resource as xr
+from x import signals as xs
+
+
+class BaseResourceTest(unittest.TestCase):
+    def setUp(self):
+        self.storage_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.storage_dir)
+        db.clear()
+        xs.clear()
+
+    def make_resource_meta(self, meta_yaml):
+        meta = yaml.load(meta_yaml)
+
+        path = os.path.join(self.storage_dir, meta['id'])
+        os.makedirs(path)
+        with open(os.path.join(path, 'meta.yaml'), 'w') as f:
+            f.write(meta_yaml)
+
+        return path
+
+    def create_resource(self, name, src, args):
+        dst = os.path.join(self.storage_dir, 'rs', name)
+        os.makedirs(dst)
+
+        return xr.create(name, src, dst, args)
+

x/test/test_signals.py

@@ -0,0 +1,140 @@
+import unittest
+
+import base
+
+from x import signals as xs
+
+
+class TestBaseInput(base.BaseResourceTest):
+    def test_input_dict_type(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  values: {}
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'values': {'a': 1, 'b': 2}}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'values': None}
+        )
+        xs.connect(sample1, sample2)
+        self.assertItemsEqual(
+            sample1.args['values'],
+            sample2.args['values'],
+        )
+
+
+class TestListInput(base.BaseResourceTest):
+    def test_list_input_single(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+        """)
+        list_input_single_meta_dir = self.make_resource_meta("""
+id: list-input-single
+handler: ansible
+version: 1.0.0
+input:
+  ips:
+input-types:
+  ips: list
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'ip': '10.0.0.1'}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'ip': '10.0.0.2'}
+        )
+        list_input_single = self.create_resource(
+            'list-input-single', list_input_single_meta_dir, {'ips': {}}
+        )
+
+        xs.connect(sample1, list_input_single, mapping={'ip': 'ips'})
+        self.assertItemsEqual(
+            list_input_single.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+            }
+        )
+
+        xs.connect(sample2, list_input_single, mapping={'ip': 'ips'})
+        self.assertItemsEqual(
+            list_input_single.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+                'sample2': sample2.args['ip'],
+            }
+        )
+
+    def test_list_input_multi(self):
+        sample_meta_dir = self.make_resource_meta("""
+id: sample
+handler: ansible
+version: 1.0.0
+input:
+  ip:
+  port:
+        """)
+        list_input_multi_meta_dir = self.make_resource_meta("""
+id: list-input-multi
+handler: ansible
+version: 1.0.0
+input:
+  ips:
+  ports:
+input-types:
+  ips: list
+  ports: list
+        """)
+
+        sample1 = self.create_resource(
+            'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': '1000'}
+        )
+        sample2 = self.create_resource(
+            'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': '1001'}
+        )
+        list_input_multi = self.create_resource(
+            'list-input-multi', list_input_multi_meta_dir, {'ips': {}, 'ports': {}}
+        )
+
+        xs.connect(sample1, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'})
+        self.assertItemsEqual(
+            list_input_multi.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+            }
+        )
+        self.assertItemsEqual(
+            list_input_multi.args['ports'],
+            {
+                'sample1': sample1.args['port'],
+            }
+        )
+
+        xs.connect(sample2, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'})
+        self.assertItemsEqual(
+            list_input_multi.args['ips'],
+            {
+                'sample1': sample1.args['ip'],
+                'sample2': sample2.args['ip'],
+            }
+        )
+        self.assertItemsEqual(
+            list_input_multi.args['ports'],
+            {
+                'sample1': sample1.args['port'],
+                'sample2': sample2.args['port'],
+            }
+        )
+
+
+if __name__ == '__main__':
+    unittest.main()