From 3fdb11f0936c95270b71e3d62bee092a699e8a44 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 21 Apr 2015 14:51:39 +0200 Subject: [PATCH 1/7] First working version of haproxy Docker --- Vagrantfile | 7 +++++- haproxy.cfg | 26 ++++++++++++++++++++ haproxy_deployment/haproxy-deployment.yaml | 4 +-- haproxy_deployment/haproxy_deployment.py | 2 ++ x/resources/docker_container/actions/run.yml | 17 +++++++++++-- x/resources/docker_container/meta.yaml | 2 ++ x/resources/haproxy/meta.yaml | 2 ++ 7 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 haproxy.cfg diff --git a/Vagrantfile b/Vagrantfile index 61641f61..bf248c30 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -29,6 +29,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "solar-dev2" do |guest2| + guest2.vm.provision "shell", inline: init_script, privileged: true guest2.vm.network "private_network", ip: "10.0.0.3" guest2.vm.host_name = "solar-dev2" @@ -39,6 +40,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "solar-dev3" do |guest3| + guest3.vm.provision "shell", inline: init_script, privileged: true guest3.vm.network "private_network", ip: "10.0.0.4" guest3.vm.host_name = "solar-dev3" @@ -49,6 +51,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "solar-dev4" do |guest4| + guest4.vm.provision "shell", inline: init_script, privileged: true guest4.vm.network "private_network", ip: "10.0.0.5" guest4.vm.host_name = "solar-dev4" @@ -59,6 +62,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "solar-dev5" do |guest5| + guest5.vm.provision "shell", inline: init_script, privileged: true guest5.vm.network "private_network", ip: "10.0.0.6" guest5.vm.host_name = "solar-dev5" @@ -69,8 +73,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end config.vm.define "solar-dev6" do |guest6| + guest6.vm.provision "shell", inline: init_script, privileged: true guest6.vm.network "private_network", ip: "10.0.0.7" - guest6.vm.host_name = "solar-dev5" + guest6.vm.host_name = "solar-dev6" guest6.vm.provider :virtualbox do |v| v.customize ["modifyvm", :id, "--memory", 256] diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 00000000..736a5185 --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,26 @@ +global + log 127.0.0.1 local0 + log 127.0.0.1 local1 notice + maxconn 4096 + tune.ssl.default-dh-param 2048 + pidfile /var/run/haproxy.pid + user haproxy + group haproxy + daemon + stats socket /var/run/haproxy.stats level admin + ssl-default-bind-options no-sslv3 +defaults + log global + mode http + option redispatch + option httplog + option dontlognull + option forwardfor + timeout connect 5000 + timeout client 50000 + timeout server 50000 +frontend default_frontend + bind 0.0.0.0:80 + default_backend default_service +backend default_service + balance roundrobin diff --git a/haproxy_deployment/haproxy-deployment.yaml b/haproxy_deployment/haproxy-deployment.yaml index bda416b5..e90a2103 100755 --- a/haproxy_deployment/haproxy-deployment.yaml +++ b/haproxy_deployment/haproxy-deployment.yaml @@ -132,11 +132,11 @@ resources: model: x/resources/docker_container args: ip: - image: haproxy + image: tutum/haproxy ssh_user: ssh_key: host_binds: - - /etc/haproxy: /vagrant/haproxy-etc + - /etc/haproxy: /etc/haproxy volume_binds: diff --git a/haproxy_deployment/haproxy_deployment.py b/haproxy_deployment/haproxy_deployment.py index 606bf1c3..4cc641dd 100644 --- a/haproxy_deployment/haproxy_deployment.py +++ b/haproxy_deployment/haproxy_deployment.py @@ -55,6 +55,8 @@ class TestHAProxyDeployment(unittest.TestCase): haproxy = db.get_resource('haproxy') self.assertEqual(node5.args['ip'], haproxy.args['ip']) + self.assertEqual(node5.args['ssh_key'], haproxy.args['ssh_key']) + self.assertEqual(node5.args['ssh_user'], haproxy.args['ssh_user']) #self.assertItemsEqual( # haproxy.args['configs'], # { diff --git a/x/resources/docker_container/actions/run.yml b/x/resources/docker_container/actions/run.yml index 90ae50dc..71af55f2 100644 --- a/x/resources/docker_container/actions/run.yml +++ b/x/resources/docker_container/actions/run.yml @@ -2,5 +2,18 @@ - hosts: [{{ ip }}] sudo: yes tasks: - - shell: docker run -d --net="host" --privileged \ - --name {{ name }} {{ image }} + - apt: name=python-pip state=present + - shell: pip install docker-py + - service: name=docker state=started + - file: path=/etc/haproxy/haproxy.cfg state=touch + - template: src=/vagrant/haproxy.cfg dest=/etc/haproxy/haproxy.cfg + - docker: + name: {{ name }} + image: {{ image }} + state: running + volumes: + {% for bind in host_binds %} + {% for src, dst in bind.items () %} + - {{ src }}:{{ dst }} + {% endfor %} + {% endfor %} diff --git a/x/resources/docker_container/meta.yaml b/x/resources/docker_container/meta.yaml index ad46d110..8a104815 100644 --- a/x/resources/docker_container/meta.yaml +++ b/x/resources/docker_container/meta.yaml @@ -6,6 +6,8 @@ input: image: host_binds: volume_binds: + ssh_user: + ssh_key: input-types: host_binds: list volume_binds: list diff --git a/x/resources/haproxy/meta.yaml b/x/resources/haproxy/meta.yaml index 136a855d..95d8773f 100644 --- a/x/resources/haproxy/meta.yaml +++ b/x/resources/haproxy/meta.yaml @@ -4,5 +4,7 @@ version: 1.0.0 input: ip: configs: + ssh_user: + ssh_key: input-types: configs: list From 98e5b3375be8f59c166719fdc905033448bac396 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 21 Apr 2015 20:15:09 +0200 Subject: [PATCH 2/7] Successful HAProxy deployment with config file - Some unit tests added --- haproxy.cfg | 30 +++- haproxy_deployment/haproxy-deployment.yaml | 59 +++++--- haproxy_deployment/haproxy_deployment.py | 42 +++++- x/db.py | 6 + x/deployment.py | 5 +- x/handlers/ansible.py | 5 +- x/resource.py | 11 +- x/resources/docker_container/actions/run.yml | 12 +- x/resources/haproxy/actions/run.yml | 20 ++- x/resources/haproxy/meta.yaml | 3 + x/resources/haproxy_config/meta.yaml | 2 + x/resources/keystone/meta.yaml | 1 + x/resources/nova/meta.yaml | 1 + x/signals.py | 11 ++ x/test/__init__.py | 1 + x/test/test_signals.py | 146 +++++++++++++++++++ 16 files changed, 308 insertions(+), 47 deletions(-) create mode 100644 x/test/__init__.py create mode 100644 x/test/test_signals.py diff --git a/haproxy.cfg b/haproxy.cfg index 736a5185..93401c17 100644 --- a/haproxy.cfg +++ b/haproxy.cfg @@ -9,6 +9,7 @@ global daemon stats socket /var/run/haproxy.stats level admin ssl-default-bind-options no-sslv3 + defaults log global mode http @@ -19,8 +20,27 @@ defaults timeout connect 5000 timeout client 50000 timeout server 50000 -frontend default_frontend - bind 0.0.0.0:80 - default_backend default_service -backend default_service - balance roundrobin + +#frontend default_frontend +# bind 0.0.0.0:80 +# default_backend default_service + +#backend default_service +# balance roundrobin + +{% for service in haproxy_services %} +listen {{ service['name'] }} 0.0.0.0:{{ service['servers'][0]['port'] }} + mode http + stats enable + stats uri /haproxy?stats + stats realm Strictly\ Private + stats auth A_Username:YourPassword + stats auth Another_User:passwd + balance roundrobin + option httpclose + option forwardfor + {% for server in service['servers'] %} + server {{ server['name'] }} {{ server['ip'] }}:{{ server['port'] }} check + {% endfor %} + +{% endfor %} diff --git a/haproxy_deployment/haproxy-deployment.yaml b/haproxy_deployment/haproxy-deployment.yaml index e90a2103..d755d17a 100755 --- a/haproxy_deployment/haproxy-deployment.yaml +++ b/haproxy_deployment/haproxy-deployment.yaml @@ -72,6 +72,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} + ports: {} ssh_user: ssh_key: @@ -111,6 +112,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} + ports: {} ssh_user: ssh_key: @@ -121,13 +123,14 @@ resources: # image: haproxy-config # export_volumes: # - haproxy-config - #- name: haproxy-config - # model: x/resources/haproxy/ - # args: - # ip: - # configs: {} - # ssh_user: - # ssh_key: + - name: haproxy-config + model: x/resources/haproxy/ + args: + ip: + configs: {} + configs_ports: {} + ssh_user: + ssh_key: - name: haproxy model: x/resources/docker_container args: @@ -135,9 +138,8 @@ resources: image: tutum/haproxy ssh_user: ssh_key: - host_binds: - - /etc/haproxy: /etc/haproxy - volume_binds: + host_binds: {} + volume_binds: {} connections: @@ -153,10 +155,12 @@ connections: receiver: haproxy_keystone_config mapping: ip: servers + port: ports - emitter: keystone2 receiver: haproxy_keystone_config mapping: ip: servers + port: ports - emitter: node3 receiver: mariadb_nova1_data @@ -170,28 +174,43 @@ connections: receiver: haproxy_nova_config mapping: ip: servers + port: ports - emitter: nova2 receiver: haproxy_nova_config mapping: ip: servers + port: ports # HAProxy config container + - emitter: node5 + receiver: haproxy-config + #- emitter: node5 # receiver: haproxy-config-container #- emitter: haproxy-config-container # receiver: haproxy-config - #- emitter: haproxy_keystone_config - # receiver: haproxy-config - # mapping: - # servers: configs - #- emitter: haproxy_nova_config - # receiver: haproxy-config - # mapping: - # servers: configs + - emitter: haproxy_keystone_config + receiver: haproxy-config + mapping: + ports: configs_ports + servers: configs + - emitter: haproxy_nova_config + receiver: haproxy-config + mapping: + ports: configs_ports + servers: configs + + - emitter: haproxy-config + receiver: haproxy + mapping: + ip: ip + ssh_user: ssh_user + ssh_key: ssh_key + config_dir: host_binds # HAProxy service - - emitter: node5 - receiver: haproxy + #- emitter: node5 + # receiver: haproxy #- emitter: haproxy-config # receiver: haproxy diff --git a/haproxy_deployment/haproxy_deployment.py b/haproxy_deployment/haproxy_deployment.py index 4cc641dd..d88e11ba 100644 --- a/haproxy_deployment/haproxy_deployment.py +++ b/haproxy_deployment/haproxy_deployment.py @@ -25,6 +25,13 @@ class TestHAProxyDeployment(unittest.TestCase): 'keystone2': keystone2.args['ip'], } ) + self.assertDictEqual( + haproxy_keystone_config.args['ports'], + { + 'keystone1': keystone1.args['port'], + 'keystone2': keystone2.args['port'], + } + ) def test_nova_config(self): node3 = db.get_resource('node3') @@ -47,23 +54,44 @@ class TestHAProxyDeployment(unittest.TestCase): 'nova2': nova2.args['ip'], } ) + self.assertDictEqual( + haproxy_nova_config.args['ports'], + { + 'nova1': nova1.args['port'], + 'nova2': nova2.args['port'], + } + ) def test_haproxy(self): node5 = db.get_resource('node5') haproxy_keystone_config = db.get_resource('haproxy_keystone_config') haproxy_nova_config = db.get_resource('haproxy_nova_config') haproxy = db.get_resource('haproxy') + haproxy_config = db.get_resource('haproxy-config') self.assertEqual(node5.args['ip'], haproxy.args['ip']) self.assertEqual(node5.args['ssh_key'], haproxy.args['ssh_key']) self.assertEqual(node5.args['ssh_user'], haproxy.args['ssh_user']) - #self.assertItemsEqual( - # haproxy.args['configs'], - # { - # 'haproxy_keystone_config': haproxy_keystone_config.args['servers'], - # 'haproxy_nova_config': haproxy_nova_config.args['servers'], - # } - #) + self.assertItemsEqual( + haproxy_config.args['configs'], + { + 'haproxy_keystone_config': haproxy_keystone_config.args['servers'], + 'haproxy_nova_config': haproxy_nova_config.args['servers'], + } + ) + self.assertItemsEqual( + haproxy_config.args['configs_ports'], + { + 'haproxy_keystone_config': haproxy_keystone_config.args['ports'], + 'haproxy_nova_config': haproxy_nova_config.args['ports'], + } + ) + self.assertItemsEqual( + { + 'haproxy-config': haproxy_config.args['config_dir'], + }, + haproxy.args['host_binds'] + ) def main(): diff --git a/x/db.py b/x/db.py index 90df6dfe..4ffcc490 100644 --- a/x/db.py +++ b/x/db.py @@ -11,3 +11,9 @@ def resource_add(key, value): def get_resource(key): return RESOURCE_DB.get(key, None) + + +def clear(): + global RESOURCE_DB + + RESOURCE_DB = {} diff --git a/x/deployment.py b/x/deployment.py index 2f2e3c8a..96bda097 100644 --- a/x/deployment.py +++ b/x/deployment.py @@ -17,9 +17,8 @@ def deploy(filename): resource_save_path = os.path.join(workdir, config['resource-save-path']) # Clean stuff first - clients_file = os.path.join(workdir, 'clients.json') - if os.path.exists(clients_file): - os.remove(clients_file) + db.clear() + xs.clear() shutil.rmtree(resource_save_path, ignore_errors=True) os.makedirs(resource_save_path) diff --git a/x/handlers/ansible.py b/x/handlers/ansible.py index 7ea9c76d..ff9640ef 100644 --- a/x/handlers/ansible.py +++ b/x/handlers/ansible.py @@ -1,6 +1,7 @@ # -*- coding: UTF-8 -*- import os import subprocess +import yaml from x.handlers.base import BaseHandler @@ -11,7 +12,9 @@ class Ansible(BaseHandler): playbook_file = self._create_playbook(resource, action_name) print 'inventory_file', inventory_file print 'playbook_file', playbook_file - subprocess.call(['ansible-playbook', '-i', inventory_file, playbook_file]) + call_args = ['ansible-playbook', '-i', inventory_file, playbook_file] + print 'EXECUTING: ', ' '.join(call_args) + subprocess.call(call_args) #def _get_connection(self, resource): # return {'ssh_user': '', diff --git a/x/resource.py b/x/resource.py index 4500e4c3..895105d1 100644 --- a/x/resource.py +++ b/x/resource.py @@ -19,7 +19,7 @@ class Resource(object): self.metadata = metadata self.actions = metadata['actions'].keys() if metadata['actions'] else None self.requires = metadata['input'].keys() - self._validate_args(args) + self._validate_args(args, metadata['input']) self.args = args self.metadata['input'] = args self.input_types = metadata.get('input-types', {}) @@ -63,10 +63,15 @@ class Resource(object): else: raise Exception('Uuups, action is not available') - def _validate_args(self, args): + def _validate_args(self, args, inputs): for req in self.requires: if req not in args: - raise Exception('Requirement `{0}` is missing in args'.format(req)) + # If metadata input is filled with a value, use it as default + # and don't report an error + if inputs.get(req): + args[req] = inputs[req] + else: + raise Exception('Requirement `{0}` is missing in args'.format(req)) # TODO: versioning def save(self): diff --git a/x/resources/docker_container/actions/run.yml b/x/resources/docker_container/actions/run.yml index 71af55f2..e02c6e39 100644 --- a/x/resources/docker_container/actions/run.yml +++ b/x/resources/docker_container/actions/run.yml @@ -5,15 +5,15 @@ - apt: name=python-pip state=present - shell: pip install docker-py - service: name=docker state=started - - file: path=/etc/haproxy/haproxy.cfg state=touch - - template: src=/vagrant/haproxy.cfg dest=/etc/haproxy/haproxy.cfg - docker: name: {{ name }} image: {{ image }} state: running volumes: - {% for bind in host_binds %} - {% for src, dst in bind.items () %} - - {{ src }}:{{ dst }} - {% endfor %} + # TODO: host_binds might need more work + # Currently it's not that trivial to pass custom src: dst here + # (when a config variable is passed here from other resource) + # so we mount it to the same directory as on host + {% for emitter, bind in host_binds.items() %} + - {{ bind }}:{{ bind }} {% endfor %} diff --git a/x/resources/haproxy/actions/run.yml b/x/resources/haproxy/actions/run.yml index e223fe8f..56dbdac8 100644 --- a/x/resources/haproxy/actions/run.yml +++ b/x/resources/haproxy/actions/run.yml @@ -1,6 +1,22 @@ # TODO - hosts: [{{ ip }}] sudo: yes + vars: + config_dir: {{ config_dir }} + haproxy_ip: {{ ip }} + haproxy_services: + {% for service, servers in configs.items() %} + - name: {{ service }} + servers: + {% for name, ip in servers.items() %} + - name: {{ name }} + ip: {{ ip }} + port: {{ configs_ports[service][name] }} + {% endfor %} + {% endfor %} tasks: - - shell: docker run -d --net="host" --privileged \ - --name {{ name }} {{ image }} + - apt: name=python-pip state=present + - shell: pip install docker-py + - service: name=docker state=started + - file: path=/etc/haproxy/haproxy.cfg state=touch + - template: src=/vagrant/haproxy.cfg dest=/etc/haproxy/haproxy.cfg diff --git a/x/resources/haproxy/meta.yaml b/x/resources/haproxy/meta.yaml index 95d8773f..5459ee0f 100644 --- a/x/resources/haproxy/meta.yaml +++ b/x/resources/haproxy/meta.yaml @@ -3,8 +3,11 @@ handler: ansible version: 1.0.0 input: ip: + config_dir: /etc/haproxy configs: + configs_ports: ssh_user: ssh_key: input-types: configs: list + configs_ports: list diff --git a/x/resources/haproxy_config/meta.yaml b/x/resources/haproxy_config/meta.yaml index 0c767796..48b2d171 100644 --- a/x/resources/haproxy_config/meta.yaml +++ b/x/resources/haproxy_config/meta.yaml @@ -2,6 +2,8 @@ id: haproxy_config handler: ansible version: 1.0.0 input: + ports: servers: input-types: + ports: list servers: list diff --git a/x/resources/keystone/meta.yaml b/x/resources/keystone/meta.yaml index 80bcb326..404e2c7e 100644 --- a/x/resources/keystone/meta.yaml +++ b/x/resources/keystone/meta.yaml @@ -3,4 +3,5 @@ handler: ansible version: 1.0.0 input: ip: + port: 5000 image: garland/docker-openstack-keystone diff --git a/x/resources/nova/meta.yaml b/x/resources/nova/meta.yaml index e6a861c0..0591a410 100644 --- a/x/resources/nova/meta.yaml +++ b/x/resources/nova/meta.yaml @@ -3,4 +3,5 @@ handler: ansible version: 1.0.0 input: ip: + port: 8774 image: # TODO diff --git a/x/signals.py b/x/signals.py index fdb7c2ac..e5827962 100644 --- a/x/signals.py +++ b/x/signals.py @@ -2,6 +2,7 @@ from collections import defaultdict import itertools import networkx as nx +import os import db @@ -12,6 +13,16 @@ CLIENTS_CONFIG_KEY = 'clients-data-file' CLIENTS = utils.read_config_file(CLIENTS_CONFIG_KEY) +def clear(): + global CLIENTS + + CLIENTS = {} + + path = utils.read_config()[CLIENTS_CONFIG_KEY] + if os.path.exists(path): + os.remove(path) + + def guess_mapping(emitter, receiver): """Guess connection mapping between emitter and receiver. diff --git a/x/test/__init__.py b/x/test/__init__.py new file mode 100644 index 00000000..4bf2011a --- /dev/null +++ b/x/test/__init__.py @@ -0,0 +1 @@ +__author__ = 'przemek' diff --git a/x/test/test_signals.py b/x/test/test_signals.py new file mode 100644 index 00000000..acf91a85 --- /dev/null +++ b/x/test/test_signals.py @@ -0,0 +1,146 @@ +import os +import shutil +import tempfile +import unittest +import yaml + +from x import db +from x import resource as xr +from x import signals as xs + + +class TestListInput(unittest.TestCase): + def setUp(self): + self.storage_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.storage_dir) + db.clear() + xs.clear() + + def make_resource_meta(self, meta_yaml): + meta = yaml.load(meta_yaml) + + path = os.path.join(self.storage_dir, meta['id']) + os.makedirs(path) + with open(os.path.join(path, 'meta.yaml'), 'w') as f: + f.write(meta_yaml) + + return path + + def create_resource(self, name, src, args): + dst = os.path.join(self.storage_dir, 'rs', name) + os.makedirs(dst) + + return xr.create(name, src, dst, args) + + def test_list_input_single(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + ip: + """) + list_input_single_meta_dir = self.make_resource_meta(""" +id: list-input-single +handler: ansible +version: 1.0.0 +input: + ips: +input-types: + ips: list + """) + + sample1 = self.create_resource( + 'sample1', sample_meta_dir, {'ip': '10.0.0.1'} + ) + sample2 = self.create_resource( + 'sample2', sample_meta_dir, {'ip': '10.0.0.2'} + ) + list_input_single = self.create_resource( + 'list-input-single', list_input_single_meta_dir, {'ips': {}} + ) + + xs.connect(sample1, list_input_single, mapping={'ip': 'ips'}) + self.assertItemsEqual( + list_input_single.args['ips'], + { + 'sample1': sample1.args['ip'], + } + ) + + xs.connect(sample2, list_input_single, mapping={'ip': 'ips'}) + self.assertItemsEqual( + list_input_single.args['ips'], + { + 'sample1': sample1.args['ip'], + 'sample2': sample2.args['ip'], + } + ) + + + def test_list_input_multi(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + ip: + port: + """) + list_input_multi_meta_dir = self.make_resource_meta(""" +id: list-input-multi +handler: ansible +version: 1.0.0 +input: + ips: + ports: +input-types: + ips: list + ports: list + """) + + sample1 = self.create_resource( + 'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': '1000'} + ) + sample2 = self.create_resource( + 'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': '1001'} + ) + list_input_multi = self.create_resource( + 'list-input-multi', list_input_multi_meta_dir, {'ips': {}, 'ports': {}} + ) + + xs.connect(sample1, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'}) + self.assertItemsEqual( + list_input_multi.args['ips'], + { + 'sample1': sample1.args['ip'], + } + ) + self.assertItemsEqual( + list_input_multi.args['ports'], + { + 'sample1': sample1.args['port'], + } + ) + + xs.connect(sample2, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'}) + self.assertItemsEqual( + list_input_multi.args['ips'], + { + 'sample1': sample1.args['ip'], + 'sample2': sample2.args['ip'], + } + ) + self.assertItemsEqual( + list_input_multi.args['ports'], + { + 'sample1': sample1.args['port'], + 'sample2': sample2.args['port'], + } + ) + + +if __name__ == '__main__': + unittest.main() From a8ae9cdf263e8f5ec1b85397282507368238a164 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 22 Apr 2015 09:13:17 +0200 Subject: [PATCH 3/7] HAProxy deployment - Added ports mapping to Docker container - tests refactored --- haproxy_deployment/haproxy-deployment.yaml | 22 +-------- haproxy_deployment/haproxy_deployment.py | 4 ++ x/resources/docker_container/actions/run.yml | 20 +++++--- x/resources/docker_container/meta.yaml | 4 +- x/test/base.py | 36 ++++++++++++++ x/test/test_signals.py | 52 +++++++++----------- 6 files changed, 81 insertions(+), 57 deletions(-) create mode 100644 x/test/base.py diff --git a/haproxy_deployment/haproxy-deployment.yaml b/haproxy_deployment/haproxy-deployment.yaml index d755d17a..aac1d3b3 100755 --- a/haproxy_deployment/haproxy-deployment.yaml +++ b/haproxy_deployment/haproxy-deployment.yaml @@ -116,13 +116,6 @@ resources: ssh_user: ssh_key: - #- name: haproxy-config-container - # model: x/resources/data_container/ - # args: - # ip: - # image: haproxy-config - # export_volumes: - # - haproxy-config - name: haproxy-config model: x/resources/haproxy/ args: @@ -136,6 +129,7 @@ resources: args: ip: image: tutum/haproxy + ports: {} ssh_user: ssh_key: host_binds: {} @@ -185,10 +179,6 @@ connections: - emitter: node5 receiver: haproxy-config - #- emitter: node5 - # receiver: haproxy-config-container - #- emitter: haproxy-config-container - # receiver: haproxy-config - emitter: haproxy_keystone_config receiver: haproxy-config mapping: @@ -204,15 +194,7 @@ connections: receiver: haproxy mapping: ip: ip + configs_ports: ports ssh_user: ssh_user ssh_key: ssh_key config_dir: host_binds - - # HAProxy service - #- emitter: node5 - # receiver: haproxy - - #- emitter: haproxy-config - # receiver: haproxy - # mapping: - diff --git a/haproxy_deployment/haproxy_deployment.py b/haproxy_deployment/haproxy_deployment.py index d88e11ba..47dca192 100644 --- a/haproxy_deployment/haproxy_deployment.py +++ b/haproxy_deployment/haproxy_deployment.py @@ -92,6 +92,10 @@ class TestHAProxyDeployment(unittest.TestCase): }, haproxy.args['host_binds'] ) + self.assertItemsEqual( + haproxy.args['ports'], + haproxy_config.args['configs_ports'], + ) def main(): diff --git a/x/resources/docker_container/actions/run.yml b/x/resources/docker_container/actions/run.yml index e02c6e39..f767054e 100644 --- a/x/resources/docker_container/actions/run.yml +++ b/x/resources/docker_container/actions/run.yml @@ -9,11 +9,17 @@ name: {{ name }} image: {{ image }} state: running + ports: + {% for name, ports_dict in ports.items() %} + # TODO: this is ugly + # {{ name }} + - {{ ports_dict.values()[0] }}:{{ ports_dict.values()[0] }} + {% endfor %} volumes: - # TODO: host_binds might need more work - # Currently it's not that trivial to pass custom src: dst here - # (when a config variable is passed here from other resource) - # so we mount it to the same directory as on host - {% for emitter, bind in host_binds.items() %} - - {{ bind }}:{{ bind }} - {% endfor %} + # TODO: host_binds might need more work + # Currently it's not that trivial to pass custom src: dst here + # (when a config variable is passed here from other resource) + # so we mount it to the same directory as on host + {% for emitter, bind in host_binds.items() %} + - {{ bind }}:{{ bind }} + {% endfor %} diff --git a/x/resources/docker_container/meta.yaml b/x/resources/docker_container/meta.yaml index 8a104815..182c872c 100644 --- a/x/resources/docker_container/meta.yaml +++ b/x/resources/docker_container/meta.yaml @@ -3,11 +3,13 @@ handler: ansible version: 1.0.0 input: ip: - image: + image: + ports: host_binds: volume_binds: ssh_user: ssh_key: input-types: + ports: host_binds: list volume_binds: list diff --git a/x/test/base.py b/x/test/base.py new file mode 100644 index 00000000..6dbceb97 --- /dev/null +++ b/x/test/base.py @@ -0,0 +1,36 @@ +import os +import shutil +import tempfile +import unittest +import yaml + +from x import db +from x import resource as xr +from x import signals as xs + + +class BaseResourceTest(unittest.TestCase): + def setUp(self): + self.storage_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.storage_dir) + db.clear() + xs.clear() + + def make_resource_meta(self, meta_yaml): + meta = yaml.load(meta_yaml) + + path = os.path.join(self.storage_dir, meta['id']) + os.makedirs(path) + with open(os.path.join(path, 'meta.yaml'), 'w') as f: + f.write(meta_yaml) + + return path + + def create_resource(self, name, src, args): + dst = os.path.join(self.storage_dir, 'rs', name) + os.makedirs(dst) + + return xr.create(name, src, dst, args) + diff --git a/x/test/test_signals.py b/x/test/test_signals.py index acf91a85..26c49ded 100644 --- a/x/test/test_signals.py +++ b/x/test/test_signals.py @@ -1,39 +1,34 @@ -import os -import shutil -import tempfile import unittest -import yaml -from x import db -from x import resource as xr +import base + from x import signals as xs -class TestListInput(unittest.TestCase): - def setUp(self): - self.storage_dir = tempfile.mkdtemp() +class TestBaseInput(base.BaseResourceTest): + def test_input_dict_type(self): + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + values: {} + """) - def tearDown(self): - shutil.rmtree(self.storage_dir) - db.clear() - xs.clear() + sample1 = self.create_resource( + 'sample1', sample_meta_dir, {'values': {'a': 1, 'b': 2}} + ) + sample2 = self.create_resource( + 'sample2', sample_meta_dir, {'values': None} + ) + xs.connect(sample1, sample2) + self.assertItemsEqual( + sample1.args['values'], + sample2.args['values'], + ) - def make_resource_meta(self, meta_yaml): - meta = yaml.load(meta_yaml) - - path = os.path.join(self.storage_dir, meta['id']) - os.makedirs(path) - with open(os.path.join(path, 'meta.yaml'), 'w') as f: - f.write(meta_yaml) - - return path - - def create_resource(self, name, src, args): - dst = os.path.join(self.storage_dir, 'rs', name) - os.makedirs(dst) - - return xr.create(name, src, dst, args) +class TestListInput(base.BaseResourceTest): def test_list_input_single(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -79,7 +74,6 @@ input-types: } ) - def test_list_input_multi(self): sample_meta_dir = self.make_resource_meta(""" id: sample From f18971c092e2234ab2a143fa4b3cef98ad3d064a Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 22 Apr 2015 10:02:19 +0200 Subject: [PATCH 4/7] Add listen_port to haproxy_config This simplifies port rendering logic in docker container --- haproxy.cfg | 2 +- haproxy_deployment/haproxy-deployment.yaml | 7 ++++++- haproxy_deployment/haproxy_deployment.py | 17 ++++++++++++----- x/resources/docker_container/actions/run.yml | 5 ++--- x/resources/haproxy/actions/run.yml | 1 + x/resources/haproxy/meta.yaml | 2 ++ x/resources/haproxy_config/meta.yaml | 1 + 7 files changed, 25 insertions(+), 10 deletions(-) diff --git a/haproxy.cfg b/haproxy.cfg index 93401c17..ea258a27 100644 --- a/haproxy.cfg +++ b/haproxy.cfg @@ -29,7 +29,7 @@ defaults # balance roundrobin {% for service in haproxy_services %} -listen {{ service['name'] }} 0.0.0.0:{{ service['servers'][0]['port'] }} +listen {{ service['name'] }} 0.0.0.0:{{ service['listen_port'] }} mode http stats enable stats uri /haproxy?stats diff --git a/haproxy_deployment/haproxy-deployment.yaml b/haproxy_deployment/haproxy-deployment.yaml index aac1d3b3..cd50dfc7 100755 --- a/haproxy_deployment/haproxy-deployment.yaml +++ b/haproxy_deployment/haproxy-deployment.yaml @@ -72,6 +72,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} + port: 5000 ports: {} ssh_user: ssh_key: @@ -112,6 +113,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} + port: 8774 ports: {} ssh_user: ssh_key: @@ -120,6 +122,7 @@ resources: model: x/resources/haproxy/ args: ip: + listen_ports: {} configs: {} configs_ports: {} ssh_user: @@ -182,11 +185,13 @@ connections: - emitter: haproxy_keystone_config receiver: haproxy-config mapping: + port: listen_ports ports: configs_ports servers: configs - emitter: haproxy_nova_config receiver: haproxy-config mapping: + port: listen_ports ports: configs_ports servers: configs @@ -194,7 +199,7 @@ connections: receiver: haproxy mapping: ip: ip - configs_ports: ports + listen_ports: ports ssh_user: ssh_user ssh_key: ssh_key config_dir: host_binds diff --git a/haproxy_deployment/haproxy_deployment.py b/haproxy_deployment/haproxy_deployment.py index 47dca192..21e32585 100644 --- a/haproxy_deployment/haproxy_deployment.py +++ b/haproxy_deployment/haproxy_deployment.py @@ -72,29 +72,36 @@ class TestHAProxyDeployment(unittest.TestCase): self.assertEqual(node5.args['ip'], haproxy.args['ip']) self.assertEqual(node5.args['ssh_key'], haproxy.args['ssh_key']) self.assertEqual(node5.args['ssh_user'], haproxy.args['ssh_user']) - self.assertItemsEqual( + self.assertDictEqual( haproxy_config.args['configs'], { 'haproxy_keystone_config': haproxy_keystone_config.args['servers'], 'haproxy_nova_config': haproxy_nova_config.args['servers'], } ) - self.assertItemsEqual( + self.assertDictEqual( haproxy_config.args['configs_ports'], { 'haproxy_keystone_config': haproxy_keystone_config.args['ports'], 'haproxy_nova_config': haproxy_nova_config.args['ports'], } ) - self.assertItemsEqual( + self.assertDictEqual( + haproxy_config.args['listen_ports'], + { + 'haproxy_keystone_config': haproxy_keystone_config.args['port'], + 'haproxy_nova_config': haproxy_nova_config.args['port'], + } + ) + self.assertDictEqual( { 'haproxy-config': haproxy_config.args['config_dir'], }, haproxy.args['host_binds'] ) - self.assertItemsEqual( + self.assertDictEqual( haproxy.args['ports'], - haproxy_config.args['configs_ports'], + haproxy_config.args['listen_ports'], ) diff --git a/x/resources/docker_container/actions/run.yml b/x/resources/docker_container/actions/run.yml index f767054e..f8ceab11 100644 --- a/x/resources/docker_container/actions/run.yml +++ b/x/resources/docker_container/actions/run.yml @@ -10,10 +10,9 @@ image: {{ image }} state: running ports: - {% for name, ports_dict in ports.items() %} - # TODO: this is ugly + {% for name, port in ports.items() %} # {{ name }} - - {{ ports_dict.values()[0] }}:{{ ports_dict.values()[0] }} + - {{ port }}:{{ port }} {% endfor %} volumes: # TODO: host_binds might need more work diff --git a/x/resources/haproxy/actions/run.yml b/x/resources/haproxy/actions/run.yml index 56dbdac8..7c4a9b35 100644 --- a/x/resources/haproxy/actions/run.yml +++ b/x/resources/haproxy/actions/run.yml @@ -7,6 +7,7 @@ haproxy_services: {% for service, servers in configs.items() %} - name: {{ service }} + listen_port: {{ listen_ports[service] }} servers: {% for name, ip in servers.items() %} - name: {{ name }} diff --git a/x/resources/haproxy/meta.yaml b/x/resources/haproxy/meta.yaml index 5459ee0f..e919f9af 100644 --- a/x/resources/haproxy/meta.yaml +++ b/x/resources/haproxy/meta.yaml @@ -4,10 +4,12 @@ version: 1.0.0 input: ip: config_dir: /etc/haproxy + listen_ports: configs: configs_ports: ssh_user: ssh_key: input-types: + listen_ports: list configs: list configs_ports: list diff --git a/x/resources/haproxy_config/meta.yaml b/x/resources/haproxy_config/meta.yaml index 48b2d171..593d41f1 100644 --- a/x/resources/haproxy_config/meta.yaml +++ b/x/resources/haproxy_config/meta.yaml @@ -2,6 +2,7 @@ id: haproxy_config handler: ansible version: 1.0.0 input: + port: ports: servers: input-types: From 8b2e7259caafc79ccdeb85cc57daab1956c7e114 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 22 Apr 2015 10:14:00 +0200 Subject: [PATCH 5/7] HAProxy config listen_port name fix --- haproxy_deployment/haproxy-deployment.yaml | 8 ++++---- haproxy_deployment/haproxy_deployment.py | 4 ++-- x/resources/haproxy_config/meta.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/haproxy_deployment/haproxy-deployment.yaml b/haproxy_deployment/haproxy-deployment.yaml index cd50dfc7..78b57a10 100755 --- a/haproxy_deployment/haproxy-deployment.yaml +++ b/haproxy_deployment/haproxy-deployment.yaml @@ -72,7 +72,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} - port: 5000 + listen_port: 5000 ports: {} ssh_user: ssh_key: @@ -113,7 +113,7 @@ resources: model: x/resources/haproxy_config/ args: servers: {} - port: 8774 + listen_port: 8774 ports: {} ssh_user: ssh_key: @@ -185,13 +185,13 @@ connections: - emitter: haproxy_keystone_config receiver: haproxy-config mapping: - port: listen_ports + listen_port: listen_ports ports: configs_ports servers: configs - emitter: haproxy_nova_config receiver: haproxy-config mapping: - port: listen_ports + listen_port: listen_ports ports: configs_ports servers: configs diff --git a/haproxy_deployment/haproxy_deployment.py b/haproxy_deployment/haproxy_deployment.py index 21e32585..edf44cb9 100644 --- a/haproxy_deployment/haproxy_deployment.py +++ b/haproxy_deployment/haproxy_deployment.py @@ -89,8 +89,8 @@ class TestHAProxyDeployment(unittest.TestCase): self.assertDictEqual( haproxy_config.args['listen_ports'], { - 'haproxy_keystone_config': haproxy_keystone_config.args['port'], - 'haproxy_nova_config': haproxy_nova_config.args['port'], + 'haproxy_keystone_config': haproxy_keystone_config.args['listen_port'], + 'haproxy_nova_config': haproxy_nova_config.args['listen_port'], } ) self.assertDictEqual( diff --git a/x/resources/haproxy_config/meta.yaml b/x/resources/haproxy_config/meta.yaml index 593d41f1..3ad9355f 100644 --- a/x/resources/haproxy_config/meta.yaml +++ b/x/resources/haproxy_config/meta.yaml @@ -2,7 +2,7 @@ id: haproxy_config handler: ansible version: 1.0.0 input: - port: + listen_port: ports: servers: input-types: From 067922e71ef02656c24365c910e28330db830b66 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 22 Apr 2015 10:23:59 +0200 Subject: [PATCH 6/7] Use graphviz to render graph instead of matplotlib It looks better --- cli.py | 24 +++++++++++++++--------- main.yml | 3 ++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cli.py b/cli.py index d10dcbee..0477d01b 100644 --- a/cli.py +++ b/cli.py @@ -1,10 +1,11 @@ import click import json -import matplotlib -matplotlib.use('Agg') # don't show windows -import matplotlib.pyplot as plt +#import matplotlib +#matplotlib.use('Agg') # don't show windows +#import matplotlib.pyplot as plt import networkx as nx import os +import subprocess from x import actions as xa from x import deployment as xd @@ -146,12 +147,17 @@ def init_cli_connections(): @click.command() def graph(): g = xs.connection_graph() - pos = nx.spring_layout(g) - nx.draw_networkx_nodes(g, pos) - nx.draw_networkx_edges(g, pos, arrows=True) - nx.draw_networkx_labels(g, pos) - plt.axis('off') - plt.savefig('graph.png') + + nx.write_dot(g, 'graph.dot') + subprocess.call(['dot', '-Tps', 'graph.dot', '-o', 'graph.ps']) + + # Matplotlib + #pos = nx.spring_layout(g) + #nx.draw_networkx_nodes(g, pos) + #nx.draw_networkx_edges(g, pos, arrows=True) + #nx.draw_networkx_labels(g, pos) + #plt.axis('off') + #plt.savefig('graph.png') connections.add_command(graph) diff --git a/main.yml b/main.yml index e5876ba2..b5673080 100644 --- a/main.yml +++ b/main.yml @@ -16,7 +16,8 @@ - shell: pip install -r /vagrant/requirements.txt # Graph drawing - - apt: name=python-matplotlib state=present + #- apt: name=python-matplotlib state=present + - apt: name=python-graphviz state=present # Setup development env for solar #- shell: python setup.py develop chdir=/vagrant/solar From c5c8f2d754eb4b8aee5d52efe09c23cd402b8c4f Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 22 Apr 2015 10:42:25 +0200 Subject: [PATCH 7/7] Generated detailed connection graph, also output to PNG --- cli.py | 5 +++-- x/signals.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cli.py b/cli.py index 0477d01b..c0f72eef 100644 --- a/cli.py +++ b/cli.py @@ -146,10 +146,11 @@ def init_cli_connections(): # TODO: this requires graphing libraries @click.command() def graph(): - g = xs.connection_graph() + #g = xs.connection_graph() + g = xs.detailed_connection_graph() nx.write_dot(g, 'graph.dot') - subprocess.call(['dot', '-Tps', 'graph.dot', '-o', 'graph.ps']) + subprocess.call(['dot', '-Tpng', 'graph.dot', '-o', 'graph.png']) # Matplotlib #pos = nx.spring_layout(g) diff --git a/x/signals.py b/x/signals.py index e5827962..7e6270c7 100644 --- a/x/signals.py +++ b/x/signals.py @@ -166,3 +166,17 @@ def connection_graph(): ) return g + + +def detailed_connection_graph(): + g = nx.MultiDiGraph() + + for emitter_name, destination_values in CLIENTS.items(): + for emitter_input, receivers in CLIENTS[emitter_name].items(): + for receiver_name, receiver_input in receivers: + label = emitter_input + if emitter_input != receiver_input: + label = '{}:{}'.format(emitter_input, receiver_input) + g.add_edge(emitter_name, receiver_name, label=label) + + return g