From 0e4c07515baa98f7e7303c71366c7026f6ada83c Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 24 Jul 2015 13:30:51 +0300 Subject: [PATCH 01/11] Add resource for lxc_hosts and lxc_container --- resources/lxc_container/actions/run.yml | 19 +++++++++++++++++++ resources/lxc_container/meta.yaml | 0 resources/lxc_host/actions/run.yml | 4 ++++ resources/lxc_host/meta.yaml | 14 ++++++++++++++ 4 files changed, 37 insertions(+) create mode 100644 resources/lxc_container/actions/run.yml create mode 100644 resources/lxc_container/meta.yaml create mode 100644 resources/lxc_host/actions/run.yml create mode 100644 resources/lxc_host/meta.yaml diff --git a/resources/lxc_container/actions/run.yml b/resources/lxc_container/actions/run.yml new file mode 100644 index 00000000..58e390bc --- /dev/null +++ b/resources/lxc_container/actions/run.yml @@ -0,0 +1,19 @@ + +- hosts: '*' + sudo: yes + gather_facts: false + vars: + ansible_ssh_host: 10.0.3.2 + container_address: 10.0.3.2 + container_name: test1 + container_networks: + management_address: 10.0.3.2 + bridge: lxcbr0 + interface: eth1 + netmask: 255.255.255.0 + type: veth + physical_host: 10.0.0.3 + properties: + container_release: trusty + roles: + - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } \ No newline at end of file diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml new file mode 100644 index 00000000..e69de29b diff --git a/resources/lxc_host/actions/run.yml b/resources/lxc_host/actions/run.yml new file mode 100644 index 00000000..62f7e31f --- /dev/null +++ b/resources/lxc_host/actions/run.yml @@ -0,0 +1,4 @@ +- hosts: '*' + sudo: yes + roles: + - { role: "lxc_hosts", tags: [ "lxc-host", "host-setup" ] } \ No newline at end of file diff --git a/resources/lxc_host/meta.yaml b/resources/lxc_host/meta.yaml new file mode 100644 index 00000000..f2f5fad8 --- /dev/null +++ b/resources/lxc_host/meta.yaml @@ -0,0 +1,14 @@ +id: lxc_host +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: \ No newline at end of file From 9dcf2bc4a3c48eadc184a706e7c6b514a3932c7d Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 24 Jul 2015 18:45:26 +0300 Subject: [PATCH 02/11] Lxc container resource --- resources/lxc_container/actions/run.yml | 33 +++++++++++++------ .../lxc_container/actions/setup_roles.yml | 10 ++++++ resources/lxc_container/meta.yaml | 23 +++++++++++++ 3 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 resources/lxc_container/actions/setup_roles.yml diff --git a/resources/lxc_container/actions/run.yml b/resources/lxc_container/actions/run.yml index 58e390bc..f0b6f14f 100644 --- a/resources/lxc_container/actions/run.yml +++ b/resources/lxc_container/actions/run.yml @@ -2,18 +2,31 @@ - hosts: '*' sudo: yes gather_facts: false + # this is default variables, they will be overwritten by resource one vars: - ansible_ssh_host: 10.0.3.2 - container_address: 10.0.3.2 - container_name: test1 + ansible_ssh_host: 10.0.0.4 + container_name: test2 + inventory_hostname: test2 container_networks: - management_address: 10.0.3.2 - bridge: lxcbr0 - interface: eth1 - netmask: 255.255.255.0 - type: veth - physical_host: 10.0.0.3 + mgmt: + address: 172.18.10.5 + bridge: br-test0 + bridge_address: 172.18.10.253/24 + interface: eth1 + netmask: 255.255.255.0 + type: veth + physical_host: 10.0.0.4 properties: container_release: trusty + pre_tasks: + - shell: pip install git+https://github.com/lxc/python2-lxc.git#egg=lxc + - shell: ip l add {{item.value.bridge}} type bridge + with_dict: container_networks + ignore_errors: true + - shell: ip l set {{item.value.bridge}} up + with_dict: container_networks + - shell: ip a add dev {{item.value.bridge}} {{item.value.bridge_address}} + with_dict: container_networks + ignore_errors: true roles: - - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } \ No newline at end of file + - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } diff --git a/resources/lxc_container/actions/setup_roles.yml b/resources/lxc_container/actions/setup_roles.yml new file mode 100644 index 00000000..ef6accf6 --- /dev/null +++ b/resources/lxc_container/actions/setup_roles.yml @@ -0,0 +1,10 @@ + +- stat: path=/tmp/os-ansible-deployment + register: repo +- git: repo=https://github.com/stackforge/os-ansible-deployment.git + dest=/tmp/os-ansible-deployment + when: repo.stat.exists == False +- shell: cp -r /tmp/os-ansible-deployment/playbooks/roles/{{item}} /etc/ansible/roles/ + with_items: + - lxc_container_create + - lxc_container_destroy diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml index e69de29b..8b23d0e1 100644 --- a/resources/lxc_container/meta.yaml +++ b/resources/lxc_container/meta.yaml @@ -0,0 +1,23 @@ +id: lxc_container +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + ansible_ssh_host: + schema: str! + value: + container_address: + schema: str! + value: + container_name: + schema: str! + value: \ No newline at end of file From f6b7cf784374fd40f014d5c0f909cb2ffb3f4b3c Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Fri, 24 Jul 2015 23:16:32 +0300 Subject: [PATCH 03/11] Add SVN provider for downloading dependencies Git wont allow you to download only one directory from github, (actually there is a way - with sparse download, but it is complicated), but svn provides quite simple and user friendly way of downloading directory from github --- resources/lxc_container/meta.yaml | 7 ++++++- resources/lxc_host/meta.yaml | 8 +++++++- solar/solar/core/handlers/ansible_playbook.py | 8 ++++++++ solar/solar/core/provider.py | 20 +++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml index 8b23d0e1..eb2c3ddb 100644 --- a/resources/lxc_container/meta.yaml +++ b/resources/lxc_container/meta.yaml @@ -20,4 +20,9 @@ input: value: container_name: schema: str! - value: \ No newline at end of file + value: + roles: + schema: [{value: str}] + value: + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_container_create + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_container_destroy diff --git a/resources/lxc_host/meta.yaml b/resources/lxc_host/meta.yaml index f2f5fad8..10951ac6 100644 --- a/resources/lxc_host/meta.yaml +++ b/resources/lxc_host/meta.yaml @@ -11,4 +11,10 @@ input: value: ssh_user: schema: str! - value: \ No newline at end of file + value: + roles: + schema: [{value: str}] + value: + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_hosts + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/pip_install + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/apt_package_pinning diff --git a/solar/solar/core/handlers/ansible_playbook.py b/solar/solar/core/handlers/ansible_playbook.py index 06a5dd7b..6eed64ef 100644 --- a/solar/solar/core/handlers/ansible_playbook.py +++ b/solar/solar/core/handlers/ansible_playbook.py @@ -9,10 +9,15 @@ import ansible.constants as C from solar.core.handlers import base from solar import errors +from solar.core.provider import SVNProvider class AnsiblePlaybook(base.BaseHandler): + def download_roles(self, urls): + for url in urls: + SVNProvider(url, '/etc/ansible/roles').run() + def action(self, resource, action): action_file = os.path.join( resource.metadata['actions_path'], @@ -22,6 +27,9 @@ class AnsiblePlaybook(base.BaseHandler): runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY) variables = resource.args_dict() + if 'roles' in variables: + self.download_roles(variables['roles']) + remote_user = variables.get('ssh_user') or C.DEFAULT_REMOTE_USER private_key_file = variables.get('ssh_key') or C.DEFAULT_PRIVATE_KEY_FILE if variables.get('ip'): diff --git a/solar/solar/core/provider.py b/solar/solar/core/provider.py index 65397a98..62fe08eb 100644 --- a/solar/solar/core/provider.py +++ b/solar/solar/core/provider.py @@ -103,3 +103,23 @@ class RemoteZipProvider(BaseProvider): self.directory = os.path.join(directory, path) else: self.directory = directory + + +class SVNProvider(BaseProvider): + """With git you cant checkout only directory from repo, + but with svn you can + """ + + def __init__(self, url, path='.'): + self.url = url + self.path = path + + def run(self): + if not os.path.exists(self.path): + os.makedirs(self.path) + + if not os.path.exists(self.url.rsplit('/', 1)[-1]): + fabric.local( + 'cd {path} && svn checkout {url}'.format( + path=self.path, + url=self.url)) From 88f2ba4725affd2ac0c4c6260c436f77adf8c409 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 3 Aug 2015 18:04:05 +0300 Subject: [PATCH 04/11] Finish lxc container and lxc host resources --- resources/lxc_container/actions/run.yml | 31 +++++++------------ .../lxc_container/actions/setup_roles.yml | 10 ------ resources/lxc_container/meta.yaml | 18 +++++++++++ resources/lxc_host/actions/run.yml | 4 ++- resources/lxc_host/meta.yaml | 3 ++ 5 files changed, 36 insertions(+), 30 deletions(-) delete mode 100644 resources/lxc_container/actions/setup_roles.yml diff --git a/resources/lxc_container/actions/run.yml b/resources/lxc_container/actions/run.yml index f0b6f14f..9b8a4b6a 100644 --- a/resources/lxc_container/actions/run.yml +++ b/resources/lxc_container/actions/run.yml @@ -1,32 +1,25 @@ - - hosts: '*' sudo: yes gather_facts: false # this is default variables, they will be overwritten by resource one vars: - ansible_ssh_host: 10.0.0.4 - container_name: test2 - inventory_hostname: test2 + ansible_ssh_host: 10.0.0.3 + physical_host: 10.0.0.3 + container_name: test3 + inventory_hostname: test3 + properties: + container_release: trusty container_networks: mgmt: - address: 172.18.10.5 + address: 172.18.10.6 bridge: br-test0 - bridge_address: 172.18.10.253/24 + bridge_address: 172.18.10.252/24 interface: eth1 netmask: 255.255.255.0 type: veth - physical_host: 10.0.0.4 - properties: - container_release: trusty + pub_key: '' pre_tasks: - - shell: pip install git+https://github.com/lxc/python2-lxc.git#egg=lxc - - shell: ip l add {{item.value.bridge}} type bridge - with_dict: container_networks - ignore_errors: true - - shell: ip l set {{item.value.bridge}} up - with_dict: container_networks - - shell: ip a add dev {{item.value.bridge}} {{item.value.bridge_address}} - with_dict: container_networks - ignore_errors: true + - set_fact: + lxc_container_ssh_key: "{{ lookup('file', pub_key) }}" roles: - - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } + - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } diff --git a/resources/lxc_container/actions/setup_roles.yml b/resources/lxc_container/actions/setup_roles.yml deleted file mode 100644 index ef6accf6..00000000 --- a/resources/lxc_container/actions/setup_roles.yml +++ /dev/null @@ -1,10 +0,0 @@ - -- stat: path=/tmp/os-ansible-deployment - register: repo -- git: repo=https://github.com/stackforge/os-ansible-deployment.git - dest=/tmp/os-ansible-deployment - when: repo.stat.exists == False -- shell: cp -r /tmp/os-ansible-deployment/playbooks/roles/{{item}} /etc/ansible/roles/ - with_items: - - lxc_container_create - - lxc_container_destroy diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml index eb2c3ddb..52d33f92 100644 --- a/resources/lxc_container/meta.yaml +++ b/resources/lxc_container/meta.yaml @@ -15,12 +15,30 @@ input: ansible_ssh_host: schema: str! value: + physical_host: + schema: str! + value: container_address: schema: str! value: container_name: schema: str! value: + inventory_hostname: + schema: str! + value: + container_networks: + schema: {} + value: + properties: + schema: {} + value: + pub_key: + schema: str! + value: + requires: + schema: str + value: roles: schema: [{value: str}] value: diff --git a/resources/lxc_host/actions/run.yml b/resources/lxc_host/actions/run.yml index 62f7e31f..64805e29 100644 --- a/resources/lxc_host/actions/run.yml +++ b/resources/lxc_host/actions/run.yml @@ -1,4 +1,6 @@ - hosts: '*' sudo: yes roles: - - { role: "lxc_hosts", tags: [ "lxc-host", "host-setup" ] } \ No newline at end of file + - { role: "lxc_hosts", tags: [ "lxc-host", "host-setup" ] } + post_tasks: + - shell: pip install git+https://github.com/lxc/python2-lxc.git#egg=lxc \ No newline at end of file diff --git a/resources/lxc_host/meta.yaml b/resources/lxc_host/meta.yaml index 10951ac6..4125e937 100644 --- a/resources/lxc_host/meta.yaml +++ b/resources/lxc_host/meta.yaml @@ -12,6 +12,9 @@ input: ssh_user: schema: str! value: + provides: + schema: str + value: infra roles: schema: [{value: str}] value: From 418066b8277d5e74002449f79bf996fa844e3e65 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 3 Aug 2015 18:04:54 +0300 Subject: [PATCH 05/11] Add container_networks, ssh_key and vxlan_mesh resources --- resources/container_networks/actions/run.yml | 22 +++++++++++++++++++ resources/container_networks/meta.yaml | 17 +++++++++++++++ resources/ssh_key/actions/run.yml | 12 ++++++++++ resources/ssh_key/meta.yaml | 23 ++++++++++++++++++++ resources/vxlan_mesh/actions/run.yml | 16 ++++++++++++++ resources/vxlan_mesh/meta.yaml | 23 ++++++++++++++++++++ 6 files changed, 113 insertions(+) create mode 100644 resources/container_networks/actions/run.yml create mode 100644 resources/container_networks/meta.yaml create mode 100644 resources/ssh_key/actions/run.yml create mode 100644 resources/ssh_key/meta.yaml create mode 100644 resources/vxlan_mesh/actions/run.yml create mode 100644 resources/vxlan_mesh/meta.yaml diff --git a/resources/container_networks/actions/run.yml b/resources/container_networks/actions/run.yml new file mode 100644 index 00000000..a7ef8042 --- /dev/null +++ b/resources/container_networks/actions/run.yml @@ -0,0 +1,22 @@ +- hosts: '*' + sudo: yes + gather_facts: false + # this is default variables, they will be overwritten by resource one + vars: + networks: + mgmt: + address: 172.18.10.6 + bridge: br-test0 + bridge_address: 172.18.10.252/24 + interface: eth1 + netmask: 255.255.255.0 + type: veth + tasks: + - shell: ip l add {{item.value.bridge}} type bridge + with_dict: networks + ignore_errors: true + - shell: ip l set {{item.value.bridge}} up + with_dict: networks + - shell: ip a add dev {{item.value.bridge}} {{item.value.bridge_address}} + with_dict: networks + ignore_errors: true diff --git a/resources/container_networks/meta.yaml b/resources/container_networks/meta.yaml new file mode 100644 index 00000000..900d8839 --- /dev/null +++ b/resources/container_networks/meta.yaml @@ -0,0 +1,17 @@ +id: container_networks +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + networks: + schema: {} + value: diff --git a/resources/ssh_key/actions/run.yml b/resources/ssh_key/actions/run.yml new file mode 100644 index 00000000..83073f9b --- /dev/null +++ b/resources/ssh_key/actions/run.yml @@ -0,0 +1,12 @@ +- hosts: '*' + sudo: yes + gather_facts: false + # this is default variables, they will be overwritten by resource one + vars: + path: /vagrant/.ssh/id_rsa + passphrase: containers + tasks: + - stat: path={{path}} + register: key + - shell: ssh-keygen -t rsa -f {{path}} -N {{passphrase}} + when: key.stat.exists == False diff --git a/resources/ssh_key/meta.yaml b/resources/ssh_key/meta.yaml new file mode 100644 index 00000000..aa480273 --- /dev/null +++ b/resources/ssh_key/meta.yaml @@ -0,0 +1,23 @@ +id: ssh_key +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + path: + schema: str! + value: + pub_path: + schema: str! + value: + passphrase: + schema: str + value: default_passphrase diff --git a/resources/vxlan_mesh/actions/run.yml b/resources/vxlan_mesh/actions/run.yml new file mode 100644 index 00000000..48f5e8d3 --- /dev/null +++ b/resources/vxlan_mesh/actions/run.yml @@ -0,0 +1,16 @@ +- hosts: '*' + sudo: yes + vars: + id: 42 + group: 239.1.10.2 + parent: eth1 + master: br-test0 + tasks: + - name: add vxlan mesh + shell: ip l add vxlan{{id}} type vxlan id {{id}} + group {{group}} dev {{parent}} + ignore_errors: true + - name: set vxlan master + shell: ip l set vxlan{{id}} master {{master}} + - name: set vxlan tunnel up + shell: ip l set vxlan{{id}} up diff --git a/resources/vxlan_mesh/meta.yaml b/resources/vxlan_mesh/meta.yaml new file mode 100644 index 00000000..411af1e0 --- /dev/null +++ b/resources/vxlan_mesh/meta.yaml @@ -0,0 +1,23 @@ +id: vxlan_mesh +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + parent: + schema: str! + value: + master: + schema: str! + value: + id: + schema: int! + value: From f1ddb454a148ad7f39ab523251aabaf4b39fb992 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Mon, 3 Aug 2015 18:05:17 +0300 Subject: [PATCH 06/11] Add example-lxc.py and several fixes --- .gitignore | 2 + example-lxc.py | 98 ++++++++++++++++++++++++++++++++++++ main.yml | 1 + solar/solar/core/provider.py | 6 +-- templates/seed_node.yml | 8 +++ 5 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 example-lxc.py create mode 100644 templates/seed_node.yml diff --git a/.gitignore b/.gitignore index 9b3c4f5d..6cf97eee 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ bootstrap/solar-master.box vagrant-settings.yml .solar_cli_uids + +.ssh/ diff --git a/example-lxc.py b/example-lxc.py new file mode 100644 index 00000000..e1fd3ed1 --- /dev/null +++ b/example-lxc.py @@ -0,0 +1,98 @@ +import click +import sys +import time + +from solar.core import actions +from solar.core import resource +from solar.core import signals +from solar.core import validation +from solar.core.resource import virtual_resource as vr +from solar import errors + +from solar.interfaces.db import get_db + + +@click.group() +def main(): + pass + + +@click.command() +def deploy(): + db = get_db() + db.clear() + signals.Connections.clear() + + node1 = vr.create('nodes', 'templates/nodes.yml', {})[0] + seed = vr.create('nodes', 'templates/seed_node.yml', {})[0] + + ssh_key = vr.create('ssh_key1', 'resources/ssh_key', { + 'path': '/vagrant/.ssh/id_rsa', + 'pub_path': '/vagrant/.ssh/id_rsa.pub' + })[0] + signals.connect(seed, ssh_key) + + cnets1 = vr.create('cnets1', 'resources/container_networks', { + 'networks': + {'mgmt': { + 'bridge': 'br-int53', + 'bridge_address': '172.18.11.254/24' + }} + })[0] + cnets2 = vr.create('cnets2', 'resources/container_networks', { + 'networks': + {'mgmt': { + 'bridge': 'br-int53', + 'bridge_address': '172.18.11.253/24' + }} + })[0] + signals.connect(seed, cnets1) + signals.connect(node1, cnets2) + + vxlan_mesh1 = vr.create('vxlan_mesh1', 'resources/vxlan_mesh', { + 'id': 53, + 'parent': 'eth1', + 'master': 'br-int53' + })[0] + vxlan_mesh2 = vr.create('vxlan_mesh2', 'resources/vxlan_mesh', { + 'id': 53, + 'parent': 'eth1', + 'master': 'br-int53' + })[0] + # seed node should be connected anyway, because we need to be able to ssh + # into containers from any node + signals.connect(seed, vxlan_mesh1) + signals.connect(node1, vxlan_mesh2) + + lxc_infra1 = vr.create('lxc_infra1', 'resources/lxc_host', {})[0] + signals.connect(node1, lxc_infra1) + + lxc_host1 = vr.create('lxc_host1', 'resources/lxc_container', { + 'container_name': 'test13', + 'inventory_hostname': 'test13', + 'properties': + {'container_release': 'trusty'}, + 'container_networks': + {'mgmt': { + 'address': '172.18.11.2', # address for container + 'bridge': 'br-int53', # bridge to attach veth pair + 'bridge_address': '172.18.11.253/24', + 'interface': 'eth1', # interface name in container + 'netmask': '255.255.255.0', + 'type': 'veth'}} + })[0] + signals.connect(node1, lxc_host1, { + 'ip': ['ansible_ssh_host', 'physical_host'], + }) + # this is a required to introduce depends on relationship between lxc infre + # and lxc container + signals.connect(lxc_infra1, lxc_host1, {'provides': 'requires'}) + signals.connect(cnets2, lxc_host1) + signals.connect(ssh_key, lxc_host1, {'pub_path': 'pub_key'}) + +main.add_command(deploy) + + +if __name__ == '__main__': + main() + diff --git a/main.yml b/main.yml index 605492dd..4410d0f4 100644 --- a/main.yml +++ b/main.yml @@ -57,6 +57,7 @@ - jq - ipython - python-pudb + - subversion # Graph drawing #- apt: name=python-matplotlib state=present diff --git a/solar/solar/core/provider.py b/solar/solar/core/provider.py index 62fe08eb..6fff07b3 100644 --- a/solar/solar/core/provider.py +++ b/solar/solar/core/provider.py @@ -117,9 +117,9 @@ class SVNProvider(BaseProvider): def run(self): if not os.path.exists(self.path): os.makedirs(self.path) - - if not os.path.exists(self.url.rsplit('/', 1)[-1]): - fabric.local( + full_path_role = os.path.join(self.path, self.url.rsplit('/', 1)[-1]) + if not os.path.exists(full_path_role): + fabric_api.local( 'cd {path} && svn checkout {url}'.format( path=self.path, url=self.url)) diff --git a/templates/seed_node.yml b/templates/seed_node.yml new file mode 100644 index 00000000..81663b77 --- /dev/null +++ b/templates/seed_node.yml @@ -0,0 +1,8 @@ +id: seed_node +resources: + - id: seed_node + from: resources/ro_node + values: + ip: '10.0.0.2' + ssh_key: '/vagrant/.vagrant/machines/solar-dev/virtualbox/private_key' + ssh_user: 'vagrant' From b92672b30e5d978a75f229ff5b71b864b6a6071a Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 4 Aug 2015 12:39:09 +0300 Subject: [PATCH 07/11] Create multiple containers and add rabbitmq resource on idx=28 --- example-lxc.py | 98 ++++++++++++++++++++++--------- resources/lxc_container/meta.yaml | 9 +++ resources/ssh_key/actions/run.yml | 4 +- resources/ssh_key/meta.yaml | 2 +- 4 files changed, 81 insertions(+), 32 deletions(-) diff --git a/example-lxc.py b/example-lxc.py index e1fd3ed1..91af5a99 100644 --- a/example-lxc.py +++ b/example-lxc.py @@ -1,22 +1,37 @@ import click -import sys -import time -from solar.core import actions -from solar.core import resource from solar.core import signals -from solar.core import validation from solar.core.resource import virtual_resource as vr -from solar import errors from solar.interfaces.db import get_db +from solar.system_log import change +from solar.cli import orch @click.group() def main(): pass +def lxc_template(idx): + return { + 'user': 'root', + 'mgmt_ip': '172.18.11.{}'.format(idx), + 'container_name': 'test{}'.format(idx), + 'inventory_hostname': 'test{}'.format(idx), + 'properties': + {'container_release': 'trusty'}, + 'container_networks': + {'mgmt': { + 'address': '172.18.11.{}'.format(idx), # address for container + 'bridge': 'br-int53', # bridge to attach veth pair + 'bridge_address': '172.18.11.253/24', + 'interface': 'eth1', # interface name in container + 'netmask': '255.255.255.0', + 'type': 'veth'}} + } + + @click.command() def deploy(): db = get_db() @@ -28,7 +43,8 @@ def deploy(): ssh_key = vr.create('ssh_key1', 'resources/ssh_key', { 'path': '/vagrant/.ssh/id_rsa', - 'pub_path': '/vagrant/.ssh/id_rsa.pub' + 'pub_path': '/vagrant/.ssh/id_rsa.pub', + 'passphrase': '', })[0] signals.connect(seed, ssh_key) @@ -67,28 +83,51 @@ def deploy(): lxc_infra1 = vr.create('lxc_infra1', 'resources/lxc_host', {})[0] signals.connect(node1, lxc_infra1) - lxc_host1 = vr.create('lxc_host1', 'resources/lxc_container', { - 'container_name': 'test13', - 'inventory_hostname': 'test13', - 'properties': - {'container_release': 'trusty'}, - 'container_networks': - {'mgmt': { - 'address': '172.18.11.2', # address for container - 'bridge': 'br-int53', # bridge to attach veth pair - 'bridge_address': '172.18.11.253/24', - 'interface': 'eth1', # interface name in container - 'netmask': '255.255.255.0', - 'type': 'veth'}} - })[0] - signals.connect(node1, lxc_host1, { - 'ip': ['ansible_ssh_host', 'physical_host'], - }) - # this is a required to introduce depends on relationship between lxc infre - # and lxc container - signals.connect(lxc_infra1, lxc_host1, {'provides': 'requires'}) - signals.connect(cnets2, lxc_host1) - signals.connect(ssh_key, lxc_host1, {'pub_path': 'pub_key'}) + lxc_hosts = range(28, 35) + hosts_map = {} + for idx in lxc_hosts: + + lxc_host_idx = vr.create( + 'lxc_host{}'.format(idx), + 'resources/lxc_container', lxc_template(idx))[0] + hosts_map[idx] = lxc_host_idx + + signals.connect(node1, lxc_host_idx, { + 'ip': ['ansible_ssh_host', 'physical_host'], + }) + # this is a required to introduce depends on relationship between lxc infre + # and lxc container + signals.connect(lxc_infra1, lxc_host_idx, {'provides': 'requires'}) + signals.connect(cnets2, lxc_host_idx) + signals.connect(ssh_key, lxc_host_idx, { + 'pub_path': 'pub_key', + 'path': 'user_key'}) + + # RABBIT + rabbitmq_service1 = vr.create('rabbitmq_service1', 'resources/rabbitmq_service/', { + 'management_port': 15672, + 'port': 5672, + })[0] + openstack_vhost = vr.create('openstack_vhost', 'resources/rabbitmq_vhost/', { + 'vhost_name': 'openstack' + })[0] + + openstack_rabbitmq_user = vr.create('openstack_rabbitmq_user', 'resources/rabbitmq_user/', { + 'user_name': 'openstack', + 'password': 'openstack_password' + })[0] + + signals.connect(hosts_map[28], rabbitmq_service1, { + 'mgmt_ip': 'ip', + 'user_key': 'ssh_key', + 'user': 'ssh_user'}) + signals.connect(rabbitmq_service1, openstack_vhost) + signals.connect(rabbitmq_service1, openstack_rabbitmq_user) + signals.connect(openstack_vhost, openstack_rabbitmq_user, { + 'vhost_name', + }) + + print change.send_to_orchestration() main.add_command(deploy) @@ -96,3 +135,4 @@ main.add_command(deploy) if __name__ == '__main__': main() +ansible-playbook --module-path /vagrant/library -i /tmp/tmp8bTQzi/tmpGUYX4Bdocker20/inventory /tmp/tmp8bTQzi/tmpGUYX4Bdocker20/runIMUDej \ No newline at end of file diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml index 52d33f92..16644fc3 100644 --- a/resources/lxc_container/meta.yaml +++ b/resources/lxc_container/meta.yaml @@ -15,6 +15,15 @@ input: ansible_ssh_host: schema: str! value: + user: + schema: str! + value: + user_key: + schema: str! + value: + mgmt_ip: + schema: str! + value: physical_host: schema: str! value: diff --git a/resources/ssh_key/actions/run.yml b/resources/ssh_key/actions/run.yml index 83073f9b..f8fc1616 100644 --- a/resources/ssh_key/actions/run.yml +++ b/resources/ssh_key/actions/run.yml @@ -4,9 +4,9 @@ # this is default variables, they will be overwritten by resource one vars: path: /vagrant/.ssh/id_rsa - passphrase: containers + passphrase: '' tasks: - stat: path={{path}} register: key - - shell: ssh-keygen -t rsa -f {{path}} -N {{passphrase}} + - shell: ssh-keygen -t rsa -f {{path}} -N "" when: key.stat.exists == False diff --git a/resources/ssh_key/meta.yaml b/resources/ssh_key/meta.yaml index aa480273..ffe71dc1 100644 --- a/resources/ssh_key/meta.yaml +++ b/resources/ssh_key/meta.yaml @@ -20,4 +20,4 @@ input: value: passphrase: schema: str - value: default_passphrase + value: From 276f58fb4fb25d852b6fa8bd6d1a86cfd64bbb42 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 5 Aug 2015 11:46:16 +0300 Subject: [PATCH 08/11] SVNProvider downloads repository in resources-directory from config --- solar/requirements.txt | 1 + solar/solar/core/handlers/ansible_playbook.py | 12 ++++++++++- solar/solar/core/provider.py | 20 ++++++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/solar/requirements.txt b/solar/requirements.txt index 1896bfa0..c4288753 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -16,3 +16,4 @@ Fabric==1.10.2 tabulate==0.7.5 ansible celery +mock diff --git a/solar/solar/core/handlers/ansible_playbook.py b/solar/solar/core/handlers/ansible_playbook.py index 6eed64ef..992207ec 100644 --- a/solar/solar/core/handlers/ansible_playbook.py +++ b/solar/solar/core/handlers/ansible_playbook.py @@ -6,17 +6,27 @@ from ansible.playbook import PlayBook from ansible import utils from ansible import callbacks import ansible.constants as C +from fabric import api as fabric_api from solar.core.handlers import base from solar import errors from solar.core.provider import SVNProvider +ROLES_PATH = '/etc/ansible/roles' + + class AnsiblePlaybook(base.BaseHandler): def download_roles(self, urls): + if not os.path.exists(ROLES_PATH): + os.makedirs(ROLES_PATH) + for url in urls: - SVNProvider(url, '/etc/ansible/roles').run() + provider = SVNProvider(url) + provider.run() + fabric_api.local('cp -r {} {}'.format( + provider.repo_directory, ROLES_PATH)) def action(self, resource, action): action_file = os.path.join( diff --git a/solar/solar/core/provider.py b/solar/solar/core/provider.py index 6fff07b3..1bd569d7 100644 --- a/solar/solar/core/provider.py +++ b/solar/solar/core/provider.py @@ -110,16 +110,22 @@ class SVNProvider(BaseProvider): but with svn you can """ - def __init__(self, url, path='.'): + def __init__(self, url, path='.', base_path=None): self.url = url self.path = path + self.base_path = base_path or utils.read_config()['resources-directory'] + if path != '.': + self.directory = os.path.join(self.base_path, path) + else: + self.directory = self.base_path + self.repo_directory = os.path.join(self.directory, self.url.rsplit('/', 1)[-1]) def run(self): - if not os.path.exists(self.path): - os.makedirs(self.path) - full_path_role = os.path.join(self.path, self.url.rsplit('/', 1)[-1]) - if not os.path.exists(full_path_role): + if not os.path.exists(self.directory): + os.makedirs(self.directory) + + if not os.path.exists(self.repo_directory): fabric_api.local( - 'cd {path} && svn checkout {url}'.format( - path=self.path, + 'cd {dir} && svn checkout {url}'.format( + dir=self.directory, url=self.url)) From e879c07d4bb696bc6cffe2009672b409086833c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 3 Sep 2015 12:30:37 +0300 Subject: [PATCH 09/11] Address comments to lxc branch and fix several bugs with example-lxc.py - C.HOST_KEY_CHECKING = False for ansible_playbook.py handler - mkdir -p {{keys_dir}} added to ssh_key resource run action --- example-lxc.py | 11 +++++------ resources/ssh_key/actions/run.yml | 8 +++++--- resources/ssh_key/meta.yaml | 7 +++++-- solar/solar/core/handlers/ansible_playbook.py | 5 ++--- solar/solar/core/provider.py | 14 +++++++------- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/example-lxc.py b/example-lxc.py index 91af5a99..6f77ef02 100644 --- a/example-lxc.py +++ b/example-lxc.py @@ -42,8 +42,9 @@ def deploy(): seed = vr.create('nodes', 'templates/seed_node.yml', {})[0] ssh_key = vr.create('ssh_key1', 'resources/ssh_key', { - 'path': '/vagrant/.ssh/id_rsa', - 'pub_path': '/vagrant/.ssh/id_rsa.pub', + 'keys_dir': '/vagrant/.ssh', + 'private_key': '/vagrant/.ssh/id_rsa', + 'public_key': '/vagrant/.ssh/id_rsa.pub', 'passphrase': '', })[0] signals.connect(seed, ssh_key) @@ -100,8 +101,8 @@ def deploy(): signals.connect(lxc_infra1, lxc_host_idx, {'provides': 'requires'}) signals.connect(cnets2, lxc_host_idx) signals.connect(ssh_key, lxc_host_idx, { - 'pub_path': 'pub_key', - 'path': 'user_key'}) + 'public_key': 'pub_key', + 'private_key': 'user_key'}) # RABBIT rabbitmq_service1 = vr.create('rabbitmq_service1', 'resources/rabbitmq_service/', { @@ -134,5 +135,3 @@ main.add_command(deploy) if __name__ == '__main__': main() - -ansible-playbook --module-path /vagrant/library -i /tmp/tmp8bTQzi/tmpGUYX4Bdocker20/inventory /tmp/tmp8bTQzi/tmpGUYX4Bdocker20/runIMUDej \ No newline at end of file diff --git a/resources/ssh_key/actions/run.yml b/resources/ssh_key/actions/run.yml index f8fc1616..11423977 100644 --- a/resources/ssh_key/actions/run.yml +++ b/resources/ssh_key/actions/run.yml @@ -3,10 +3,12 @@ gather_facts: false # this is default variables, they will be overwritten by resource one vars: - path: /vagrant/.ssh/id_rsa + keys_dir: /vagrant/.ssh + private_key: /vagrant/.ssh/id_rsa passphrase: '' tasks: - - stat: path={{path}} + - shell: mkdir -p {{keys_dir}} + - stat: path={{private_key}} register: key - - shell: ssh-keygen -t rsa -f {{path}} -N "" + - shell: ssh-keygen -t rsa -f {{private_key}} -N "" when: key.stat.exists == False diff --git a/resources/ssh_key/meta.yaml b/resources/ssh_key/meta.yaml index ffe71dc1..690764c6 100644 --- a/resources/ssh_key/meta.yaml +++ b/resources/ssh_key/meta.yaml @@ -12,10 +12,13 @@ input: ssh_user: schema: str! value: - path: + keys_dir: schema: str! value: - pub_path: + private_key: + schema: str! + value: + public_key: schema: str! value: passphrase: diff --git a/solar/solar/core/handlers/ansible_playbook.py b/solar/solar/core/handlers/ansible_playbook.py index 992207ec..44ba5964 100644 --- a/solar/solar/core/handlers/ansible_playbook.py +++ b/solar/solar/core/handlers/ansible_playbook.py @@ -21,12 +21,11 @@ class AnsiblePlaybook(base.BaseHandler): def download_roles(self, urls): if not os.path.exists(ROLES_PATH): os.makedirs(ROLES_PATH) - for url in urls: provider = SVNProvider(url) provider.run() fabric_api.local('cp -r {} {}'.format( - provider.repo_directory, ROLES_PATH)) + provider.directory, ROLES_PATH)) def action(self, resource, action): action_file = os.path.join( @@ -48,7 +47,7 @@ class AnsiblePlaybook(base.BaseHandler): else: host = 'localhost' transport = 'local' - + C.HOST_KEY_CHECKING = False play = PlayBook( playbook=action_file, remote_user=remote_user, diff --git a/solar/solar/core/provider.py b/solar/solar/core/provider.py index 1bd569d7..855ce7a7 100644 --- a/solar/solar/core/provider.py +++ b/solar/solar/core/provider.py @@ -115,17 +115,17 @@ class SVNProvider(BaseProvider): self.path = path self.base_path = base_path or utils.read_config()['resources-directory'] if path != '.': - self.directory = os.path.join(self.base_path, path) + self.repo_directory = os.path.join(self.base_path, path) else: - self.directory = self.base_path - self.repo_directory = os.path.join(self.directory, self.url.rsplit('/', 1)[-1]) + self.repo_directory = self.base_path + self.directory = os.path.join(self.repo_directory, self.url.rsplit('/', 1)[-1]) def run(self): - if not os.path.exists(self.directory): - os.makedirs(self.directory) - if not os.path.exists(self.repo_directory): + os.makedirs(self.repo_directory) + + if not os.path.exists(self.directory): fabric_api.local( 'cd {dir} && svn checkout {url}'.format( - dir=self.directory, + dir=self.repo_directory, url=self.url)) From 63b223e7c9a0f27c9875042d1be18a61cc3d5162 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 3 Sep 2015 15:02:10 +0200 Subject: [PATCH 10/11] LXC: add svn to bootstrap, add comment on how to run to example-lxc.py --- bootstrap/playbooks/tasks/base.yml | 1 + example-lxc.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/bootstrap/playbooks/tasks/base.yml b/bootstrap/playbooks/tasks/base.yml index 2debca36..813bb09d 100644 --- a/bootstrap/playbooks/tasks/base.yml +++ b/bootstrap/playbooks/tasks/base.yml @@ -6,6 +6,7 @@ apt: name={{ item }} state=present with_items: - git + - subversion - python-mock - python-keystoneclient - python-mysqldb diff --git a/example-lxc.py b/example-lxc.py index 6f77ef02..b9bb734d 100644 --- a/example-lxc.py +++ b/example-lxc.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python + +# To run: +# example-lxc.py deploy +# solar changes stage +# solar changes process +# solar orch run-once last +# watch 'solar orch report last' + import click from solar.core import signals From ccdf3c0affe9623bb952041404e4f7ca3cf437dd Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 3 Sep 2015 15:06:01 +0200 Subject: [PATCH 11/11] main.yaml: subversion is in bootstrap already --- main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/main.yml b/main.yml index 4410d0f4..605492dd 100644 --- a/main.yml +++ b/main.yml @@ -57,7 +57,6 @@ - jq - ipython - python-pudb - - subversion # Graph drawing #- apt: name=python-matplotlib state=present