Deployment from YAML file, added tests for haproxy

cli.py

@@ -6,6 +6,8 @@ import matplotlib.pyplot as plt
 import networkx as nx
 import os
 
+from x import actions as xa
+from x import deployment as xd
 from x import resource as xr
 from x import signals as xs
 
@@ -22,6 +24,16 @@ def init_cli_resource():
 
     cli.add_command(resource)
 
+    @click.command()
+    @click.argument('resource_path')
+    @click.argument('action_name')
+    def action(action_name, resource_path):
+        print 'action', resource_path, action_name
+        r = xr.load(resource_path)
+        xa.resource_action(r, action_name)
+
+    resource.add_command(action)
+
     @click.command()
     @click.argument('name')
     @click.argument('base_path')
@@ -144,9 +156,20 @@ def init_cli_connections():
     connections.add_command(graph)
 
 
+def init_cli_deployment_config():
+    @click.command()
+    @click.argument('filepath')
+    def deploy(filepath):
+        print 'Deploying from file {}'.format(filepath)
+        xd.deploy(filepath)
+
+    cli.add_command(deploy)
+
+
 if __name__ == '__main__':
     init_cli_resource()
     init_cli_connect()
     init_cli_connections()
+    init_cli_deployment_config()
 
     cli()

haproxy_deployment/__init__.py

@@ -0,0 +1 @@
+__author__ = 'przemek'

haproxy_deployment/haproxy-deployment.yaml

@@ -0,0 +1,166 @@
+# HAProxy deployment with MariaDB, Keystone and Nova
+
+workdir: /vagrant
+resource-save-path: rs/
+test-suite: haproxy_deployment.haproxy_deployment
+
+resources:
+  - name: node1
+    model: x/resources/ro_node/
+    args:
+      ip: 10.0.0.3
+      ssh_key: /vagrant/tmp/keys/ssh_private
+      ssh_user: vagrant
+  - name: node2
+    model: x/resources/ro_node/
+    args:
+      ip: 10.0.0.4
+      ssh_key: /vagrant/tmp/keys/ssh_private
+      ssh_user: vagrant
+  - name: node3
+    model: x/resources/ro_node/
+    args:
+      ip: 10.0.0.5
+      ssh_key: /vagrant/tmp/keys/ssh_private
+      ssh_user: vagrant
+  - name: node4
+    model: x/resources/ro_node/
+    args:
+      ip: 10.0.0.6
+      ssh_key: /vagrant/tmp/keys/ssh_private
+      ssh_user: vagrant
+  - name: node5
+    model: x/resources/ro_node/
+    args:
+      ip: 10.0.0.7
+      ssh_key: /vagrant/tmp/keys/ssh_private
+      ssh_user: vagrant
+
+  - name: mariadb_keystone1_data
+    model: x/resources/data_container/
+    args:
+      image: mariadb
+      export_volumes:
+        - /var/lib/mysql
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: mariadb_keystone2_data
+    model: x/resources/data_container/
+    args:
+      image: mariadb
+      export_volumes:
+        - /var/lib/mysql
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: keystone1
+    model: x/resources/keystone/
+    args:
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: keystone2
+    model: x/resources/keystone/
+    args:
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: haproxy_keystone_config
+    model: x/resources/haproxy_config/
+    args:
+      servers: {}
+      ssh_user:
+      ssh_key:
+
+  - name: mariadb_nova1_data
+    model: x/resources/data_container/
+    args:
+      image: mariadb
+      export_volumes:
+        - /var/lib/mysql
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: mariadb_nova2_data
+    model: x/resources/data_container/
+    args:
+      image: mariadb
+      export_volumes:
+        - /var/lib/mysql
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: nova1
+    model: x/resources/nova/
+    args:
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: nova2
+    model: x/resources/nova/
+    args:
+      ip:
+      ssh_user:
+      ssh_key:
+  - name: haproxy_nova_config
+    model: x/resources/haproxy_config/
+    args:
+      servers: {}
+      ssh_user:
+      ssh_key:
+
+  - name: haproxy
+    model: x/resources/haproxy/
+    args:
+      ip:
+      configs: {}
+      ssh_user:
+      ssh_key:
+
+
+connections:
+  - emitter: node1
+    receiver: mariadb_keystone1_data
+  - emitter: node2
+    receiver: mariadb_keystone2_data
+  - emitter: mariadb_keystone1_data
+    receiver: keystone1
+  - emitter: mariadb_keystone2_data
+    receiver: keystone2
+  - emitter: keystone1
+    receiver: haproxy_keystone_config
+    mapping:
+      ip: servers
+  - emitter: keystone2
+    receiver: haproxy_keystone_config
+    mapping:
+      ip: servers
+
+  - emitter: node3
+    receiver: mariadb_nova1_data
+  - emitter: node4
+    receiver: mariadb_nova2_data
+  - emitter: mariadb_nova1_data
+    receiver: nova1
+  - emitter: mariadb_nova2_data
+    receiver: nova2
+  - emitter: nova1
+    receiver: haproxy_nova_config
+    mapping:
+      ip: servers
+  - emitter: nova2
+    receiver: haproxy_nova_config
+    mapping:
+      ip: servers
+
+  - emitter: node5
+    receiver: haproxy
+  - emitter: haproxy_keystone_config
+    receiver: haproxy
+    mapping:
+      server: configs
+  - emitter: haproxy_nova_config
+    receiver: haproxy
+    mapping:
+      server: configs

haproxy_deployment/haproxy_deployment.py

@@ -0,0 +1,61 @@
+import unittest
+
+from x import db
+
+
+class TestHAProxyDeployment(unittest.TestCase):
+    def test_keystone_config(self):
+        node1 = db.get_resource('node1')
+        node2 = db.get_resource('node2')
+        keystone1 = db.get_resource('keystone1')
+        keystone2 = db.get_resource('keystone2')
+
+        self.assertEqual(keystone1.args['ip'], node1.args['ip'])
+        self.assertEqual(keystone2.args['ip'], node2.args['ip'])
+
+    def test_haproxy_keystone_config(self):
+        keystone1 = db.get_resource('keystone1')
+        keystone2 = db.get_resource('keystone2')
+        haproxy_keystone_config = db.get_resource('haproxy_keystone_config')
+
+        self.assertDictEqual(
+            haproxy_keystone_config.args['servers'],
+            {
+                'keystone1': keystone1.args['ip'],
+                'keystone2': keystone2.args['ip'],
+            }
+        )
+
+    def test_nova_config(self):
+        node3 = db.get_resource('node3')
+        node4 = db.get_resource('node4')
+        nova1 = db.get_resource('nova1')
+        nova2 = db.get_resource('nova2')
+
+        self.assertEqual(nova1.args['ip'], node3.args['ip'])
+        self.assertEqual(nova2.args['ip'], node4.args['ip'])
+
+    def test_haproxy_nova_config(self):
+        nova1 = db.get_resource('nova1')
+        nova2 = db.get_resource('nova2')
+        haproxy_nova_config = db.get_resource('haproxy_nova_config')
+
+        self.assertDictEqual(
+            haproxy_nova_config.args['servers'],
+            {
+                'nova1': nova1.args['ip'],
+                'nova2': nova2.args['ip'],
+            }
+        )
+
+    def test_haproxy(self):
+        node5 = db.get_resource('node5')
+        haproxy = db.get_resource('haproxy')
+
+        self.assertEqual(node5.args['ip'], haproxy.args['ip'])
+
+
+def main():
+    loader = unittest.TestLoader()
+    suite = loader.loadTestsFromTestCase(TestHAProxyDeployment)
+    unittest.TextTestRunner().run(suite)

x/deployment.py

@@ -0,0 +1,50 @@
+# Deploying stuff from YAML definition
+
+import imp
+import os
+import shutil
+import yaml
+
+#from x import actions as xa
+from x import db
+from x import resource as xr
+from x import signals as xs
+
+
+def deploy(filename):
+    with open(filename) as f:
+        config = yaml.load(f)
+
+    workdir = config['workdir']
+    resource_save_path = os.path.join(workdir, config['resource-save-path'])
+
+    # Clean stuff first
+    clients_file = os.path.join(workdir, 'clients.json')
+    if os.path.exists(clients_file):
+        os.remove(clients_file)
+    shutil.rmtree(resource_save_path, ignore_errors=True)
+    os.makedirs(resource_save_path)
+
+    # Create resources first
+    for resource_definition in config['resources']:
+        name = resource_definition['name']
+        model = os.path.join(workdir, resource_definition['model'])
+        args = resource_definition.get('args', {})
+        print 'Creating ', name, model, resource_save_path, args
+        xr.create(name, model, resource_save_path, args=args)
+
+    # Create resource connections
+    for connection in config['connections']:
+        emitter = db.get_resource(connection['emitter'])
+        receiver = db.get_resource(connection['receiver'])
+        mapping = config.get('mapping')
+        print 'Connecting ', emitter.name, receiver.name, mapping
+        xs.connect(emitter, receiver, mapping=mapping)
+
+    # Run all tests
+    if 'test-suite' in config:
+        #test_suite_path = os.path.join(workdir, config['test-suite'])
+        print 'Running tests from {}'.format(config['test-suite'])
+        #test_suite = imp.load_source('main', test_suite_path)
+        test_suite = __import__(config['test-suite'], {}, {}, ['main'])
+        test_suite.main()