diff --git a/.gitignore b/.gitignore index 9b3c4f5d..6cf97eee 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ bootstrap/solar-master.box vagrant-settings.yml .solar_cli_uids + +.ssh/ diff --git a/bootstrap/playbooks/tasks/base.yml b/bootstrap/playbooks/tasks/base.yml index 2debca36..813bb09d 100644 --- a/bootstrap/playbooks/tasks/base.yml +++ b/bootstrap/playbooks/tasks/base.yml @@ -6,6 +6,7 @@ apt: name={{ item }} state=present with_items: - git + - subversion - python-mock - python-keystoneclient - python-mysqldb diff --git a/example-lxc.py b/example-lxc.py new file mode 100644 index 00000000..b9bb734d --- /dev/null +++ b/example-lxc.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +# To run: +# example-lxc.py deploy +# solar changes stage +# solar changes process +# solar orch run-once last +# watch 'solar orch report last' + +import click + +from solar.core import signals +from solar.core.resource import virtual_resource as vr + +from solar.interfaces.db import get_db + +from solar.system_log import change +from solar.cli import orch + +@click.group() +def main(): + pass + + +def lxc_template(idx): + return { + 'user': 'root', + 'mgmt_ip': '172.18.11.{}'.format(idx), + 'container_name': 'test{}'.format(idx), + 'inventory_hostname': 'test{}'.format(idx), + 'properties': + {'container_release': 'trusty'}, + 'container_networks': + {'mgmt': { + 'address': '172.18.11.{}'.format(idx), # address for container + 'bridge': 'br-int53', # bridge to attach veth pair + 'bridge_address': '172.18.11.253/24', + 'interface': 'eth1', # interface name in container + 'netmask': '255.255.255.0', + 'type': 'veth'}} + } + + +@click.command() +def deploy(): + db = get_db() + db.clear() + signals.Connections.clear() + + node1 = vr.create('nodes', 'templates/nodes.yml', {})[0] + seed = vr.create('nodes', 'templates/seed_node.yml', {})[0] + + ssh_key = vr.create('ssh_key1', 'resources/ssh_key', { + 'keys_dir': '/vagrant/.ssh', + 'private_key': '/vagrant/.ssh/id_rsa', + 'public_key': '/vagrant/.ssh/id_rsa.pub', + 'passphrase': '', + })[0] + signals.connect(seed, ssh_key) + + cnets1 = vr.create('cnets1', 'resources/container_networks', { + 'networks': + {'mgmt': { + 'bridge': 'br-int53', + 'bridge_address': '172.18.11.254/24' + }} + })[0] + cnets2 = vr.create('cnets2', 'resources/container_networks', { + 'networks': + {'mgmt': { + 'bridge': 'br-int53', + 'bridge_address': '172.18.11.253/24' + }} + })[0] + signals.connect(seed, cnets1) + signals.connect(node1, cnets2) + + vxlan_mesh1 = vr.create('vxlan_mesh1', 'resources/vxlan_mesh', { + 'id': 53, + 'parent': 'eth1', + 'master': 'br-int53' + })[0] + vxlan_mesh2 = vr.create('vxlan_mesh2', 'resources/vxlan_mesh', { + 'id': 53, + 'parent': 'eth1', + 'master': 'br-int53' + })[0] + # seed node should be connected anyway, because we need to be able to ssh + # into containers from any node + signals.connect(seed, vxlan_mesh1) + signals.connect(node1, vxlan_mesh2) + + lxc_infra1 = vr.create('lxc_infra1', 'resources/lxc_host', {})[0] + signals.connect(node1, lxc_infra1) + + lxc_hosts = range(28, 35) + hosts_map = {} + for idx in lxc_hosts: + + lxc_host_idx = vr.create( + 'lxc_host{}'.format(idx), + 'resources/lxc_container', lxc_template(idx))[0] + hosts_map[idx] = lxc_host_idx + + signals.connect(node1, lxc_host_idx, { + 'ip': ['ansible_ssh_host', 'physical_host'], + }) + # this is a required to introduce depends on relationship between lxc infre + # and lxc container + signals.connect(lxc_infra1, lxc_host_idx, {'provides': 'requires'}) + signals.connect(cnets2, lxc_host_idx) + signals.connect(ssh_key, lxc_host_idx, { + 'public_key': 'pub_key', + 'private_key': 'user_key'}) + + # RABBIT + rabbitmq_service1 = vr.create('rabbitmq_service1', 'resources/rabbitmq_service/', { + 'management_port': 15672, + 'port': 5672, + })[0] + openstack_vhost = vr.create('openstack_vhost', 'resources/rabbitmq_vhost/', { + 'vhost_name': 'openstack' + })[0] + + openstack_rabbitmq_user = vr.create('openstack_rabbitmq_user', 'resources/rabbitmq_user/', { + 'user_name': 'openstack', + 'password': 'openstack_password' + })[0] + + signals.connect(hosts_map[28], rabbitmq_service1, { + 'mgmt_ip': 'ip', + 'user_key': 'ssh_key', + 'user': 'ssh_user'}) + signals.connect(rabbitmq_service1, openstack_vhost) + signals.connect(rabbitmq_service1, openstack_rabbitmq_user) + signals.connect(openstack_vhost, openstack_rabbitmq_user, { + 'vhost_name', + }) + + print change.send_to_orchestration() + +main.add_command(deploy) + + +if __name__ == '__main__': + main() diff --git a/resources/container_networks/actions/run.yml b/resources/container_networks/actions/run.yml new file mode 100644 index 00000000..a7ef8042 --- /dev/null +++ b/resources/container_networks/actions/run.yml @@ -0,0 +1,22 @@ +- hosts: '*' + sudo: yes + gather_facts: false + # this is default variables, they will be overwritten by resource one + vars: + networks: + mgmt: + address: 172.18.10.6 + bridge: br-test0 + bridge_address: 172.18.10.252/24 + interface: eth1 + netmask: 255.255.255.0 + type: veth + tasks: + - shell: ip l add {{item.value.bridge}} type bridge + with_dict: networks + ignore_errors: true + - shell: ip l set {{item.value.bridge}} up + with_dict: networks + - shell: ip a add dev {{item.value.bridge}} {{item.value.bridge_address}} + with_dict: networks + ignore_errors: true diff --git a/resources/container_networks/meta.yaml b/resources/container_networks/meta.yaml new file mode 100644 index 00000000..900d8839 --- /dev/null +++ b/resources/container_networks/meta.yaml @@ -0,0 +1,17 @@ +id: container_networks +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + networks: + schema: {} + value: diff --git a/resources/lxc_container/actions/run.yml b/resources/lxc_container/actions/run.yml new file mode 100644 index 00000000..9b8a4b6a --- /dev/null +++ b/resources/lxc_container/actions/run.yml @@ -0,0 +1,25 @@ +- hosts: '*' + sudo: yes + gather_facts: false + # this is default variables, they will be overwritten by resource one + vars: + ansible_ssh_host: 10.0.0.3 + physical_host: 10.0.0.3 + container_name: test3 + inventory_hostname: test3 + properties: + container_release: trusty + container_networks: + mgmt: + address: 172.18.10.6 + bridge: br-test0 + bridge_address: 172.18.10.252/24 + interface: eth1 + netmask: 255.255.255.0 + type: veth + pub_key: '' + pre_tasks: + - set_fact: + lxc_container_ssh_key: "{{ lookup('file', pub_key) }}" + roles: + - { role: "lxc_container_create", tags: [ "lxc-container-create" ] } diff --git a/resources/lxc_container/meta.yaml b/resources/lxc_container/meta.yaml new file mode 100644 index 00000000..16644fc3 --- /dev/null +++ b/resources/lxc_container/meta.yaml @@ -0,0 +1,55 @@ +id: lxc_container +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + ansible_ssh_host: + schema: str! + value: + user: + schema: str! + value: + user_key: + schema: str! + value: + mgmt_ip: + schema: str! + value: + physical_host: + schema: str! + value: + container_address: + schema: str! + value: + container_name: + schema: str! + value: + inventory_hostname: + schema: str! + value: + container_networks: + schema: {} + value: + properties: + schema: {} + value: + pub_key: + schema: str! + value: + requires: + schema: str + value: + roles: + schema: [{value: str}] + value: + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_container_create + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_container_destroy diff --git a/resources/lxc_host/actions/run.yml b/resources/lxc_host/actions/run.yml new file mode 100644 index 00000000..64805e29 --- /dev/null +++ b/resources/lxc_host/actions/run.yml @@ -0,0 +1,6 @@ +- hosts: '*' + sudo: yes + roles: + - { role: "lxc_hosts", tags: [ "lxc-host", "host-setup" ] } + post_tasks: + - shell: pip install git+https://github.com/lxc/python2-lxc.git#egg=lxc \ No newline at end of file diff --git a/resources/lxc_host/meta.yaml b/resources/lxc_host/meta.yaml new file mode 100644 index 00000000..4125e937 --- /dev/null +++ b/resources/lxc_host/meta.yaml @@ -0,0 +1,23 @@ +id: lxc_host +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + provides: + schema: str + value: infra + roles: + schema: [{value: str}] + value: + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/lxc_hosts + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/pip_install + - https://github.com/stackforge/os-ansible-deployment/trunk/playbooks/roles/apt_package_pinning diff --git a/resources/ssh_key/actions/run.yml b/resources/ssh_key/actions/run.yml new file mode 100644 index 00000000..11423977 --- /dev/null +++ b/resources/ssh_key/actions/run.yml @@ -0,0 +1,14 @@ +- hosts: '*' + sudo: yes + gather_facts: false + # this is default variables, they will be overwritten by resource one + vars: + keys_dir: /vagrant/.ssh + private_key: /vagrant/.ssh/id_rsa + passphrase: '' + tasks: + - shell: mkdir -p {{keys_dir}} + - stat: path={{private_key}} + register: key + - shell: ssh-keygen -t rsa -f {{private_key}} -N "" + when: key.stat.exists == False diff --git a/resources/ssh_key/meta.yaml b/resources/ssh_key/meta.yaml new file mode 100644 index 00000000..690764c6 --- /dev/null +++ b/resources/ssh_key/meta.yaml @@ -0,0 +1,26 @@ +id: ssh_key +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + keys_dir: + schema: str! + value: + private_key: + schema: str! + value: + public_key: + schema: str! + value: + passphrase: + schema: str + value: diff --git a/resources/vxlan_mesh/actions/run.yml b/resources/vxlan_mesh/actions/run.yml new file mode 100644 index 00000000..48f5e8d3 --- /dev/null +++ b/resources/vxlan_mesh/actions/run.yml @@ -0,0 +1,16 @@ +- hosts: '*' + sudo: yes + vars: + id: 42 + group: 239.1.10.2 + parent: eth1 + master: br-test0 + tasks: + - name: add vxlan mesh + shell: ip l add vxlan{{id}} type vxlan id {{id}} + group {{group}} dev {{parent}} + ignore_errors: true + - name: set vxlan master + shell: ip l set vxlan{{id}} master {{master}} + - name: set vxlan tunnel up + shell: ip l set vxlan{{id}} up diff --git a/resources/vxlan_mesh/meta.yaml b/resources/vxlan_mesh/meta.yaml new file mode 100644 index 00000000..411af1e0 --- /dev/null +++ b/resources/vxlan_mesh/meta.yaml @@ -0,0 +1,23 @@ +id: vxlan_mesh +handler: ansible_playbook +version: 1.0.0 +actions: +input: + ip: + schema: str! + value: + ssh_key: + schema: str! + value: + ssh_user: + schema: str! + value: + parent: + schema: str! + value: + master: + schema: str! + value: + id: + schema: int! + value: diff --git a/solar/requirements.txt b/solar/requirements.txt index 1896bfa0..c4288753 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -16,3 +16,4 @@ Fabric==1.10.2 tabulate==0.7.5 ansible celery +mock diff --git a/solar/solar/core/handlers/ansible_playbook.py b/solar/solar/core/handlers/ansible_playbook.py index 06a5dd7b..44ba5964 100644 --- a/solar/solar/core/handlers/ansible_playbook.py +++ b/solar/solar/core/handlers/ansible_playbook.py @@ -6,13 +6,27 @@ from ansible.playbook import PlayBook from ansible import utils from ansible import callbacks import ansible.constants as C +from fabric import api as fabric_api from solar.core.handlers import base from solar import errors +from solar.core.provider import SVNProvider + + +ROLES_PATH = '/etc/ansible/roles' class AnsiblePlaybook(base.BaseHandler): + def download_roles(self, urls): + if not os.path.exists(ROLES_PATH): + os.makedirs(ROLES_PATH) + for url in urls: + provider = SVNProvider(url) + provider.run() + fabric_api.local('cp -r {} {}'.format( + provider.directory, ROLES_PATH)) + def action(self, resource, action): action_file = os.path.join( resource.metadata['actions_path'], @@ -22,6 +36,9 @@ class AnsiblePlaybook(base.BaseHandler): runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY) variables = resource.args_dict() + if 'roles' in variables: + self.download_roles(variables['roles']) + remote_user = variables.get('ssh_user') or C.DEFAULT_REMOTE_USER private_key_file = variables.get('ssh_key') or C.DEFAULT_PRIVATE_KEY_FILE if variables.get('ip'): @@ -30,7 +47,7 @@ class AnsiblePlaybook(base.BaseHandler): else: host = 'localhost' transport = 'local' - + C.HOST_KEY_CHECKING = False play = PlayBook( playbook=action_file, remote_user=remote_user, diff --git a/solar/solar/core/provider.py b/solar/solar/core/provider.py index 65397a98..855ce7a7 100644 --- a/solar/solar/core/provider.py +++ b/solar/solar/core/provider.py @@ -103,3 +103,29 @@ class RemoteZipProvider(BaseProvider): self.directory = os.path.join(directory, path) else: self.directory = directory + + +class SVNProvider(BaseProvider): + """With git you cant checkout only directory from repo, + but with svn you can + """ + + def __init__(self, url, path='.', base_path=None): + self.url = url + self.path = path + self.base_path = base_path or utils.read_config()['resources-directory'] + if path != '.': + self.repo_directory = os.path.join(self.base_path, path) + else: + self.repo_directory = self.base_path + self.directory = os.path.join(self.repo_directory, self.url.rsplit('/', 1)[-1]) + + def run(self): + if not os.path.exists(self.repo_directory): + os.makedirs(self.repo_directory) + + if not os.path.exists(self.directory): + fabric_api.local( + 'cd {dir} && svn checkout {url}'.format( + dir=self.repo_directory, + url=self.url)) diff --git a/templates/seed_node.yml b/templates/seed_node.yml new file mode 100644 index 00000000..81663b77 --- /dev/null +++ b/templates/seed_node.yml @@ -0,0 +1,8 @@ +id: seed_node +resources: + - id: seed_node + from: resources/ro_node + values: + ip: '10.0.0.2' + ssh_key: '/vagrant/.vagrant/machines/solar-dev/virtualbox/private_key' + ssh_user: 'vagrant'