From 426ea64badd1f15a2293f8e8fbe883eb49e15ca9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 14:32:30 +0200 Subject: [PATCH 01/15] Allow to create list of dicts based on connections in VRs --- solar/core/resource/virtual_resource.py | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/solar/core/resource/virtual_resource.py b/solar/core/resource/virtual_resource.py index eb898680..b12a07f4 100644 --- a/solar/core/resource/virtual_resource.py +++ b/solar/core/resource/virtual_resource.py @@ -266,14 +266,17 @@ def parse_list_input(r_input, args): connections = [] assignments = {} for arg in args: - if is_connection(arg): + if isinstance(arg, dict): + n_connections, n_assign = parse_dicts_as_collection( + r_input, arg) + connections.extend(n_connections) + if n_assign: + add_assignment(assignments, r_input, n_assign) + elif is_connection(arg): c = parse_connection(r_input, arg) connections.append(c) else: - try: - assignments[r_input].append(arg) - except KeyError: - assignments[r_input] = [arg] + add_assignment(assignments, arg) return connections, assignments @@ -293,12 +296,32 @@ def parse_dict_input(r_input, args): return connections, assignments +def add_assignment(assignments, r_input, arg): + try: + assignments[r_input].append(arg) + except KeyError: + assignments[r_input] = [arg] + + def is_connection(arg): if isinstance(arg, basestring) and '::' in arg: return True return False +def parse_dicts_as_collection(child_input, arg): + connections = [] + assignments = {} + for key, value in arg.items(): + if is_connection(value): + connections.append( + parse_connection( + '{}:{}'.format(child_input, key), value)) + else: + assignments[key] = value + return connections, assignments + + def parse_connection(child_input, element): parent, parent_input = element.split('::', 1) try: From 92a02ae0c1c5b96851e799eaa3e72b054dee33c1 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 14:32:57 +0200 Subject: [PATCH 02/15] Add rsync transport to nodes --- resources/transport_rsync/meta.yaml | 23 +++++++++++++++++++++++ templates/nodes.yaml | 18 ++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 resources/transport_rsync/meta.yaml diff --git a/resources/transport_rsync/meta.yaml b/resources/transport_rsync/meta.yaml new file mode 100644 index 00000000..5cd3a417 --- /dev/null +++ b/resources/transport_rsync/meta.yaml @@ -0,0 +1,23 @@ +id: transport_rsync +input: + key: + schema: str! + value: + user: + schema: str! + value: + name: + schema: str! + value: rsync + location_id: + schema: str + value: + reverse: True + is_own: False + transports_id: + schema: str + value: + is_emit: False + port: + schema: int + value: 3579 diff --git a/templates/nodes.yaml b/templates/nodes.yaml index 3c2e75a9..5129ab49 100644 --- a/templates/nodes.yaml +++ b/templates/nodes.yaml @@ -7,13 +7,23 @@ resources: values: ssh_user: 'vagrant' ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key' + - id: rsync{{j}} + from: resources/transport_rsync + values: + user: vagrant + key: /vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key - id: transports{{j}} from: resources/transports values: - transports:key: ssh_transport{{j}}::ssh_key - transports:user: ssh_transport{{j}}::ssh_user - transports:port: ssh_transport{{j}}::ssh_port - transports:name: ssh_transport{{j}}::name + transports: + - key: ssh_transport{{j}}::ssh_key + user: ssh_transport{{j}}::ssh_user + port: ssh_transport{{j}}::ssh_port + name: ssh_transport{{j}}::name + - key: rsync{{j}}::key + name: rsync{{j}}::name + user: rsync{{j}}::user + port: rsync{{j}}::port - id: node{{j}} from: resources/ro_node values: From a3f3653ea18c7fc26489de436ae2ce38a04ad0cc Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 14:50:28 +0200 Subject: [PATCH 03/15] Adjust raw_ssh to plugable transport layer and use it by default --- solar/core/transports/bat.py | 4 ++-- solar/core/transports/ssh_raw.py | 33 ++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/solar/core/transports/bat.py b/solar/core/transports/bat.py index 3c7442a6..5843ed88 100644 --- a/solar/core/transports/bat.py +++ b/solar/core/transports/bat.py @@ -17,8 +17,8 @@ from solar.core.transports.base import RunTransport from solar.core.transports.base import SolarTransport from solar.core.transports.base import SyncTransport from solar.core.transports.rsync import RsyncSyncTransport -from solar.core.transports.ssh import SSHRunTransport from solar.core.transports.ssh import SSHSyncTransport +from solar.core.transports.ssh_raw import RawSSHRunTransport try: from solar.core.transports.solar_agent_transport import SolarAgentRunTransport # NOQA from solar.core.transports.solar_agent_transport import SolarAgentSyncTransport # NOQA @@ -42,7 +42,7 @@ KNOWN_SYNC_TRANSPORTS = { KNOWN_RUN_TRANSPORTS = { - 'ssh': SSHRunTransport + 'ssh': RawSSHRunTransport } diff --git a/solar/core/transports/ssh_raw.py b/solar/core/transports/ssh_raw.py index 33a61aec..87790913 100644 --- a/solar/core/transports/ssh_raw.py +++ b/solar/core/transports/ssh_raw.py @@ -20,19 +20,23 @@ from solar.core.transports.base import RunTransport class _RawSSHTransport(object): - def _ssh_props(self, resource): - return { - 'ssh_key': resource.args['ssh_key'].value, - 'ssh_user': resource.args['ssh_user'].value - } + def settings(self, resource): + transport = self.get_transport_data(resource) + host = resource.ip() + user = transport['user'] + port = transport['port'] + key = transport['key'] + return {'ssh_user': user, + 'ssh_key': key, + 'port': port, + 'ip': host} - def _ssh_command_host(self, resource): - return '{}@{}'.format(resource.args['ssh_user'].value, - resource.args['ip'].value) + def _ssh_command_host(self, settings): + return '{}@{}'.format(settings['ssh_user'], + settings['ip']) - def _ssh_cmd(self, resource): - props = self._ssh_props(resource) - return ('ssh', '-i', props['ssh_key']) + def _ssh_cmd(self, settings): + return ('ssh', '-i', settings['ssh_key']) class RawSSHRunTransport(RunTransport, _RawSSHTransport): @@ -54,9 +58,10 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport): remote_cmd = '\"%s\"' % ' && '.join(cmds) - ssh_cmd = self._ssh_cmd(resource) - ssh_cmd += (self._ssh_command_host(resource), remote_cmd) + settings = self.settings(resource) + ssh_cmd = self._ssh_cmd(settings) + ssh_cmd += (self._ssh_command_host(settings), remote_cmd) - log.debug("SSH CMD: %r", ssh_cmd) + log.debug("RAW SSH CMD: %r", ssh_cmd) return fabric_api.local(' '.join(ssh_cmd)) From 6b81fba3d2e45ee47621c005f3b8893ed2a2418e Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 15:03:45 +0200 Subject: [PATCH 04/15] Add test for list of connected dicts and fix flake --- solar/core/resource/virtual_resource.py | 2 +- solar/test/test_virtual_resource.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/solar/core/resource/virtual_resource.py b/solar/core/resource/virtual_resource.py index b12a07f4..5cf10b44 100644 --- a/solar/core/resource/virtual_resource.py +++ b/solar/core/resource/virtual_resource.py @@ -276,7 +276,7 @@ def parse_list_input(r_input, args): c = parse_connection(r_input, arg) connections.append(c) else: - add_assignment(assignments, arg) + add_assignment(assignments, r_input, arg) return connections, assignments diff --git a/solar/test/test_virtual_resource.py b/solar/test/test_virtual_resource.py index 997040b6..ec870095 100644 --- a/solar/test/test_virtual_resource.py +++ b/solar/test/test_virtual_resource.py @@ -202,6 +202,19 @@ def test_parse_connection_disable_events(): assert correct_connection == connection +def test_parse_list_of_connected_dicts(): + inputs = {'list': [ + {'key': 'emitter1::key'}, + {'key': 'emitter2::key'}]} + connections, assignments = vr.parse_inputs(inputs) + assert assignments == {} + assert connections == [ + {'child_input': 'list:key', 'parent_input': 'key', + 'parent': 'emitter1', 'events': None}, + {'child_input': 'list:key', 'parent_input': 'key', + 'parent': 'emitter2', 'events': None}] + + def test_setting_location(tmpdir): # XXX: make helper for it base_path = os.path.join( From af57691e2fb002b05e7d03de1c0af9fc20129c7e Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 15:45:10 +0200 Subject: [PATCH 05/15] Install rsync, ssh client and mount known_hosts to container --- Dockerfile | 4 ++++ docker-compose.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index 05bcb3db..9bef17ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,10 @@ FROM ubuntu:14.04 WORKDIR / +RUN apt-get update +RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync +RUN pip install ansible + ADD bootstrap/playbooks/celery.yaml /celery.yaml ADD resources /resources ADD templates /templates diff --git a/docker-compose.yml b/docker-compose.yml index 1317fdb8..4d499301 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ solar-celery: - /vagrant/templates:/vagrant/templates - /vagrant/resources:/vagrant/resources - /vagrant/library:/vagrant/library + - ~/.ssh/known_hosts:/root/.ssh/known_hosts environment: - REDIS_HOST=redis - REDIS_PORT=6379 From 0e4abe55bd7b569ebe7392bcd4cbbe4bb45079cc Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 15:45:56 +0200 Subject: [PATCH 06/15] Convert hosts example to use VRs with all transports --- examples/hosts_file/hosts.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/examples/hosts_file/hosts.py b/examples/hosts_file/hosts.py index 54d04583..52d029e0 100644 --- a/examples/hosts_file/hosts.py +++ b/examples/hosts_file/hosts.py @@ -10,31 +10,7 @@ from solar.dblayer.model import ModelMeta def run(): ModelMeta.remove_all() - resources = vr.create('nodes', 'templates/nodes_with_transports.yaml', {'count': 2}) - nodes = [x for x in resources if x.name.startswith('node')] - node1, node2 = nodes - - hosts1 = vr.create('hosts_file1', 'resources/hosts_file', {})[0] - hosts2 = vr.create('hosts_file2', 'resources/hosts_file', {})[0] - node1.connect(hosts1, { - 'name': 'hosts:name', - 'ip': 'hosts:ip', - }) - - node2.connect(hosts1, { - 'name': 'hosts:name', - 'ip': 'hosts:ip', - }) - - node1.connect(hosts2, { - 'name': 'hosts:name', - 'ip': 'hosts:ip', - }) - - node2.connect(hosts2, { - 'name': 'hosts:name', - 'ip': 'hosts:ip', - }) + resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2}) run() From 8f2d8651139b0a0bd7c80dc185e48a7f2104453e Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Tue, 1 Dec 2015 15:46:27 +0200 Subject: [PATCH 07/15] Change pool to gevent from preforked for worker which performs tasks --- Dockerfile | 9 ++------- bootstrap/playbooks/celery.yaml | 5 +---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9bef17ea..41aa3e3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,13 @@ FROM ubuntu:14.04 WORKDIR / -RUN apt-get update -RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync -RUN pip install ansible - ADD bootstrap/playbooks/celery.yaml /celery.yaml ADD resources /resources ADD templates /templates ADD run.sh /run.sh -RUN apt-get update -RUN apt-get install -y python python-dev python-distribute python-pip \ - libyaml-dev vim libffi-dev libssl-dev git +RUN apt-get upgrade && apt-get update +RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync libyaml-dev vim libffi-dev libssl-dev git RUN pip install ansible RUN pip install git+https://github.com/Mirantis/solar.git diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml index e0ad2c33..4baed9bc 100644 --- a/bootstrap/playbooks/celery.yaml +++ b/bootstrap/playbooks/celery.yaml @@ -15,9 +15,6 @@ - shell: celery multi stopwait 2 -A solar.orchestration.runner chdir={{ celery_dir }} tags: [stop] - - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} + - shell: celery multi start 2 -A solar.orchestration.runner -P:2 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} chdir={{ celery_dir }} tags: [master] - - shell: celery multi start 1 -A solar.orchestration.runner -Q:1 {{ hostname.stdout }} - chdir={{ celery_dir }} - tags: [slave] From 95a65a625edab010867c7d746f2203dd54a522d6 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 2 Dec 2015 17:17:54 +0200 Subject: [PATCH 08/15] Adjust celery configuration and fix ssh_raw to work with ENV --- bootstrap/playbooks/celery.yaml | 3 ++- requirements.txt | 1 + run.sh | 2 +- solar/core/transports/ssh_raw.py | 25 +++++++++++++++---------- templates/nodes_with_transports.yaml | 18 ++++++++++++++---- 5 files changed, 33 insertions(+), 16 deletions(-) diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml index 4baed9bc..53f174dd 100644 --- a/bootstrap/playbooks/celery.yaml +++ b/bootstrap/playbooks/celery.yaml @@ -15,6 +15,7 @@ - shell: celery multi stopwait 2 -A solar.orchestration.runner chdir={{ celery_dir }} tags: [stop] - - shell: celery multi start 2 -A solar.orchestration.runner -P:2 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} + - shell: celery multi start 2 -A solar.orchestration.runner -P -c:2 1000 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} chdir={{ celery_dir }} tags: [master] + diff --git a/requirements.txt b/requirements.txt index f24d026d..e4b2c596 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ pbr pydot bunch wrapt +gevent # if you want to use riak backend then riak # if you want to use sql backend then diff --git a/run.sh b/run.sh index 1d1f3cf7..6894d773 100755 --- a/run.sh +++ b/run.sh @@ -6,6 +6,6 @@ if [ -d /solar ]; then fi #used only to start celery on docker -ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave,stop +ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags install tail -f /var/run/celery/*.log diff --git a/solar/core/transports/ssh_raw.py b/solar/core/transports/ssh_raw.py index 87790913..2d73f5c0 100644 --- a/solar/core/transports/ssh_raw.py +++ b/solar/core/transports/ssh_raw.py @@ -44,19 +44,24 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport): def run(self, resource, *args, **kwargs): log.debug("RAW SSH: %s", args) - cmds = [] - cwd = kwargs.get('cwd') - if cwd: - cmds.append(('cd', cwd)) - - cmds.append(args) - + commands = [] + prefix = [] if kwargs.get('use_sudo', False): - cmds = [('sudo', ) + cmd for cmd in cmds] + prefix.append('sudo') - cmds = [' '.join(cmd) for cmd in cmds] + if kwargs.get('cwd'): + cmd = prefix + ['cd', kwargs['cwd']] + commands.append(' '.join(cmd)) - remote_cmd = '\"%s\"' % ' && '.join(cmds) + env = [] + if 'env' in kwargs: + for key, value in kwargs['env'].items(): + env.append('{}={}'.format(key, value)) + + cmd = prefix + env + list(args) + commands.append(' '.join(cmd)) + + remote_cmd = '\"%s\"' % ' && '.join(commands) settings = self.settings(resource) ssh_cmd = self._ssh_cmd(settings) diff --git a/templates/nodes_with_transports.yaml b/templates/nodes_with_transports.yaml index f0501da4..a185b195 100644 --- a/templates/nodes_with_transports.yaml +++ b/templates/nodes_with_transports.yaml @@ -6,13 +6,23 @@ resources: values: ssh_user: 'vagrant' ssh_key: '/vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key' + - id: rsync{{i}} + from: resources/transport_rsync + values: + user: vagrant + key: /vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key - id: transports{{i}} from: resources/transports values: - transports:key: ssh_transport{{i}}::ssh_key - transports:user: ssh_transport{{i}}::ssh_user - transports:port: ssh_transport{{i}}::ssh_port - transports:name: ssh_transport{{i}}::name + transports: + - key: ssh_transport{{i}}::ssh_key + user: ssh_transport{{i}}::ssh_user + port: ssh_transport{{i}}::ssh_port + name: ssh_transport{{i}}::name + - key: rsync{{i}}::key + name: rsync{{i}}::name + user: rsync{{i}}::user + port: rsync{{i}}::port - id: node{{i}} from: resources/ro_node values: From b894c1d85593fcb7033eb2fdbc99255ed7524018 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 9 Dec 2015 20:55:38 +0200 Subject: [PATCH 09/15] Reuse parse_dict_input in parse_list --- bootstrap/playbooks/celery.yaml | 2 +- solar/core/resource/virtual_resource.py | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml index 53f174dd..66b3a860 100644 --- a/bootstrap/playbooks/celery.yaml +++ b/bootstrap/playbooks/celery.yaml @@ -15,7 +15,7 @@ - shell: celery multi stopwait 2 -A solar.orchestration.runner chdir={{ celery_dir }} tags: [stop] - - shell: celery multi start 2 -A solar.orchestration.runner -P -c:2 1000 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} + - shell: celery multi start 2 -A solar.orchestration.runner -P gevent -c:2 1000 -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} chdir={{ celery_dir }} tags: [master] diff --git a/solar/core/resource/virtual_resource.py b/solar/core/resource/virtual_resource.py index 5cf10b44..f9863bbe 100644 --- a/solar/core/resource/virtual_resource.py +++ b/solar/core/resource/virtual_resource.py @@ -267,7 +267,7 @@ def parse_list_input(r_input, args): assignments = {} for arg in args: if isinstance(arg, dict): - n_connections, n_assign = parse_dicts_as_collection( + n_connections, n_assign = parse_dict_input( r_input, arg) connections.extend(n_connections) if n_assign: @@ -309,19 +309,6 @@ def is_connection(arg): return False -def parse_dicts_as_collection(child_input, arg): - connections = [] - assignments = {} - for key, value in arg.items(): - if is_connection(value): - connections.append( - parse_connection( - '{}:{}'.format(child_input, key), value)) - else: - assignments[key] = value - return connections, assignments - - def parse_connection(child_input, element): parent, parent_input = element.split('::', 1) try: From 1e199b6acdb928e0a8f290d33c490dfce59d531f Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 11:52:01 +0200 Subject: [PATCH 10/15] Use preforked pool for celery and remove gevent from reqs --- bootstrap/playbooks/celery.yaml | 2 +- docker-compose.yml | 1 + requirements.txt | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml index 66b3a860..f02da327 100644 --- a/bootstrap/playbooks/celery.yaml +++ b/bootstrap/playbooks/celery.yaml @@ -15,7 +15,7 @@ - shell: celery multi stopwait 2 -A solar.orchestration.runner chdir={{ celery_dir }} tags: [stop] - - shell: celery multi start 2 -A solar.orchestration.runner -P gevent -c:2 1000 -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} + - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} chdir={{ celery_dir }} tags: [master] diff --git a/docker-compose.yml b/docker-compose.yml index 4d499301..cc70bdad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ solar-celery: - /vagrant/resources:/vagrant/resources - /vagrant/library:/vagrant/library - ~/.ssh/known_hosts:/root/.ssh/known_hosts + - ./bootstrap/playbooks/celery.yaml:/celery.yaml environment: - REDIS_HOST=redis - REDIS_PORT=6379 diff --git a/requirements.txt b/requirements.txt index e4b2c596..f24d026d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,6 @@ pbr pydot bunch wrapt -gevent # if you want to use riak backend then riak # if you want to use sql backend then From 5a1676f08cc5a7b35055664adb23231a9bbe89a9 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 12:52:45 +0200 Subject: [PATCH 11/15] Disable host key checking for 10.0.0.* subnet --- bootstrap/playbooks/build-main.yaml | 6 ++++-- bootstrap/playbooks/files/ssh_conf | 2 ++ bootstrap/playbooks/solar.yaml | 3 +++ bootstrap/playbooks/tasks/ssh_conf.yaml | 3 +++ docker-compose.yml | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 bootstrap/playbooks/files/ssh_conf create mode 100644 bootstrap/playbooks/tasks/ssh_conf.yaml diff --git a/bootstrap/playbooks/build-main.yaml b/bootstrap/playbooks/build-main.yaml index 6308862e..8f65449b 100644 --- a/bootstrap/playbooks/build-main.yaml +++ b/bootstrap/playbooks/build-main.yaml @@ -3,10 +3,12 @@ - name: Main build script hosts: all sudo: yes + vars: + ssh_ip_mask: "10.0.0.*" tasks: - include: tasks/base.yaml - include: tasks/puppet.yaml - include: tasks/docker.yaml - #- include: celery.yaml tags=['master'] celery_dir=/var/run/celery - include: tasks/cloud_archive.yaml - #- include: tasks/mos.yaml + - include: tasks/ssh_conf.yaml + diff --git a/bootstrap/playbooks/files/ssh_conf b/bootstrap/playbooks/files/ssh_conf new file mode 100644 index 00000000..da7b2483 --- /dev/null +++ b/bootstrap/playbooks/files/ssh_conf @@ -0,0 +1,2 @@ +Host {{ssh_ip_mask}} + StrictHostKeyChecking no diff --git a/bootstrap/playbooks/solar.yaml b/bootstrap/playbooks/solar.yaml index 519c44f5..e2211caf 100644 --- a/bootstrap/playbooks/solar.yaml +++ b/bootstrap/playbooks/solar.yaml @@ -2,12 +2,15 @@ - hosts: all sudo: yes + vars: + ssh_ip_mask: "10.0.0.*" tasks: # upgrade pbr first, old version throws strange errors - shell: pip install pbr -U # Setup development env for solar - shell: pip install -e . chdir=/vagrant - shell: pip install git+git://github.com/Mirantis/solar-agent.git + - include: tasks/ssh_conf.yaml - hosts: all tasks: diff --git a/bootstrap/playbooks/tasks/ssh_conf.yaml b/bootstrap/playbooks/tasks/ssh_conf.yaml new file mode 100644 index 00000000..1c99ff94 --- /dev/null +++ b/bootstrap/playbooks/tasks/ssh_conf.yaml @@ -0,0 +1,3 @@ +--- + +- template: src=files/ssh_conf dest=/root/.ssh/config diff --git a/docker-compose.yml b/docker-compose.yml index cc70bdad..27e2f71c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ solar-celery: - /vagrant/templates:/vagrant/templates - /vagrant/resources:/vagrant/resources - /vagrant/library:/vagrant/library - - ~/.ssh/known_hosts:/root/.ssh/known_hosts + - ~/.ssh:/root/.ssh - ./bootstrap/playbooks/celery.yaml:/celery.yaml environment: - REDIS_HOST=redis From 08c040c530c2b5bc46b0e78f6e1c371471032df8 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 13:29:21 +0200 Subject: [PATCH 12/15] Add connections between nodes and hosts files --- examples/hosts_file/hosts.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/examples/hosts_file/hosts.py b/examples/hosts_file/hosts.py index 52d029e0..19342ced 100644 --- a/examples/hosts_file/hosts.py +++ b/examples/hosts_file/hosts.py @@ -12,5 +12,28 @@ def run(): resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2}) + node1, node2 = [x for x in resources if x.name.startswith('node')] + hosts1, hosts2 = [x for x in resources + if x.name.startswith('hosts_file')] + + node1.connect(hosts1, { + 'name': 'hosts:name', + 'ip': 'hosts:ip', + }) + + node2.connect(hosts1, { + 'name': 'hosts:name', + 'ip': 'hosts:ip', + }) + + node1.connect(hosts2, { + 'name': 'hosts:name', + 'ip': 'hosts:ip', + }) + + node2.connect(hosts2, { + 'name': 'hosts:name', + 'ip': 'hosts:ip', + }) run() From 99f1ccd1a2658389c029419dfd8ba44cd7f7debe Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 14:38:39 +0200 Subject: [PATCH 13/15] Improve error handling in for ssh_raw/rsync in ansible/puppet/bash --- solar/core/handlers/ansible_template.py | 15 ++++----------- solar/core/handlers/base.py | 17 +++++++++++++++++ solar/core/handlers/naive_sync.py | 3 ++- solar/core/handlers/puppet.py | 13 +++++++------ solar/core/handlers/shell.py | 14 +++++--------- solar/core/transports/base.py | 9 ++++++--- solar/core/transports/bat.py | 6 +++++- solar/core/transports/rsync.py | 8 +++----- solar/core/transports/ssh_raw.py | 5 +++-- solar/utils.py | 11 +++++++++++ 10 files changed, 63 insertions(+), 38 deletions(-) diff --git a/solar/core/handlers/ansible_template.py b/solar/core/handlers/ansible_template.py index 02d2c8e2..e0990e89 100644 --- a/solar/core/handlers/ansible_template.py +++ b/solar/core/handlers/ansible_template.py @@ -19,7 +19,6 @@ import os from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION from solar.core.handlers.base import TempFileHandler from solar.core.log import log -from solar import errors # otherwise fabric will sys.exit(1) in case of errors env.warn_only = True @@ -40,7 +39,8 @@ class AnsibleTemplate(TempFileHandler): self._copy_templates_and_scripts(resource, action_name) self.transport_sync.copy(resource, self.dst, '/tmp') self.transport_sync.copy(resource, '/vagrant/library', '/tmp') - self.transport_sync.sync_all() + sync_results = self.transport_sync.sync_all() + self.verify_sync_results(sync_results) # remote paths are not nested inside solar_local remote_playbook_file = playbook_file.replace( @@ -52,15 +52,8 @@ class AnsibleTemplate(TempFileHandler): '-i', remote_inventory_file, remote_playbook_file] log.debug('EXECUTING: %s', ' '.join(call_args)) - out = self.transport_run.run(resource, *call_args) - log.debug(out) - if out.failed: - raise errors.SolarError(out) - - # with fabric_api.shell_env(ANSIBLE_HOST_KEY_CHECKING='False'): - # out = fabric_api.local(' '.join(call_args), capture=True) - # if out.failed: - # raise errors.SolarError(out) + rst = self.transport_run.run(resource, *call_args) + self.verify_run_result(call_args, rst) def _create_inventory(self, r): directory = self.dirs[r.name] diff --git a/solar/core/handlers/base.py b/solar/core/handlers/base.py index 36e5086b..86d6cb23 100644 --- a/solar/core/handlers/base.py +++ b/solar/core/handlers/base.py @@ -21,6 +21,7 @@ import tempfile from solar.core.log import log from solar.core.transports.ssh import SSHRunTransport from solar.core.transports.ssh import SSHSyncTransport +from solar import errors from solar import utils @@ -42,6 +43,22 @@ class BaseHandler(object): self.transport_sync.bind_with(self.transport_run) self.transport_run.bind_with(self.transport_sync) + def verify_sync_results(self, results): + for result in results: + if isinstance(result, tuple) and len(result) == 3: + # TODO Include file information in result + rc, out, err = result + log.debug('RC %s OUT %s ERR %s', rc, out, err) + if rc: + raise errors.SolarError(err) + + def verify_run_result(self, cmd, result): + rc, out, err = result + log.debug('CMD %r RC %s OUT %s ERR %s', cmd, rc, out, err) + if rc: + message = 'CMD %r failed RC %s ERR %s' % (cmd, rc, err) + raise errors.SolarError(message) + def __enter__(self): return self diff --git a/solar/core/handlers/naive_sync.py b/solar/core/handlers/naive_sync.py index c06b7236..82f87095 100644 --- a/solar/core/handlers/naive_sync.py +++ b/solar/core/handlers/naive_sync.py @@ -27,4 +27,5 @@ class NaiveSync(BaseHandler): # to understand where src comes from for item in args['sources']: self.transport_sync.copy(resource, item['src'], item['dst']) - self.transport_sync.sync_all() + results = self.transport_sync.sync_all() + self.verify_sync_results(results) diff --git a/solar/core/handlers/puppet.py b/solar/core/handlers/puppet.py index c52f88b5..baf7fade 100644 --- a/solar/core/handlers/puppet.py +++ b/solar/core/handlers/puppet.py @@ -37,7 +37,8 @@ class Puppet(TempFileHandler): self.prepare_templates_and_scripts(resource, action_file, '') self.transport_sync.copy(resource, action_file, action_file_name) - self.transport_sync.sync_all() + sync_results = self.transport_sync.sync_all() + self.verify_sync_results(sync_results) cmd_args = ['puppet', 'apply', '-vd', action_file_name, @@ -46,7 +47,7 @@ class Puppet(TempFileHandler): cmd_args.append('--modulepath={}'.format( resource.args['puppet_modules'])) - cmd = self.transport_run.run( + rc, out, err = self.transport_run.run( resource, *cmd_args, env={ @@ -55,12 +56,12 @@ class Puppet(TempFileHandler): use_sudo=True, warn_only=True ) + log.debug('CMD %r RC %s OUT %s ERR %s', cmd_args, rc, out, err) # 0 - no changes, 2 - successfull changes - if cmd.return_code not in [0, 2]: + if rc not in [0, 2]: raise errors.SolarError( - 'Puppet for {} failed with {}'.format( - resource.name, cmd.return_code)) - return cmd + 'Puppet for {} failed with RC {}'.format( + resource.name, rc)) def _make_args(self, resource): return {resource.name: {'input': resource.args}} diff --git a/solar/core/handlers/shell.py b/solar/core/handlers/shell.py index da4875a6..d2c5b023 100644 --- a/solar/core/handlers/shell.py +++ b/solar/core/handlers/shell.py @@ -18,7 +18,6 @@ import os from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION from solar.core.handlers.base import TempFileHandler from solar.core.log import log -from solar import errors class Shell(TempFileHandler): @@ -34,17 +33,14 @@ class Shell(TempFileHandler): self._copy_templates_and_scripts(resource, action_name) self.transport_sync.copy(resource, self.dst, '/tmp') - self.transport_sync.sync_all() + sync_results = self.transport_sync.sync_all() + # TODO Include file information in result + self.verify_sync_results(sync_results) - cmd = self.transport_run.run( + rst = self.transport_run.run( resource, 'bash', action_file_name, use_sudo=True, warn_only=True ) - - if cmd.return_code: - raise errors.SolarError( - 'Bash execution for {} failed with {}'.format( - resource.name, cmd.return_code)) - return cmd + self.verify_run_results(['bash', action_file_name], rst) diff --git a/solar/core/transports/base.py b/solar/core/transports/base.py index 168578b5..c867d5fd 100644 --- a/solar/core/transports/base.py +++ b/solar/core/transports/base.py @@ -38,7 +38,7 @@ class Executor(object): def run(self, transport): if self.valid: - self._executor(transport) + return self._executor(transport) class SolarRunResultWrp(object): @@ -131,8 +131,10 @@ class SyncTransport(SolarTransport): self.preprocess(executor) def run_all(self): + rst = [] for executor in self.executors: - executor.run(self) + rst.append(executor.run(self)) + return rst def sync_all(self): """Syncs all @@ -142,8 +144,9 @@ class SyncTransport(SolarTransport): Could be someday changed to parallel thing. """ self.preprocess_all() - self.run_all() + rst = self.run_all() self.executors = [] # clear after all + return rst class RunTransport(SolarTransport): diff --git a/solar/core/transports/bat.py b/solar/core/transports/bat.py index 5843ed88..32c66337 100644 --- a/solar/core/transports/bat.py +++ b/solar/core/transports/bat.py @@ -60,8 +60,12 @@ class OnAll(object): def __get__(self, obj, objtype): def _inner(*args, **kwargs): + results = [] for transport in obj._used_transports: - getattr(transport, self._target)(*args, **kwargs) + rst = getattr(transport, self._target)(*args, **kwargs) + if rst: + results.extend(rst) + return results return _inner diff --git a/solar/core/transports/rsync.py b/solar/core/transports/rsync.py index 88d27b93..da03722c 100644 --- a/solar/core/transports/rsync.py +++ b/solar/core/transports/rsync.py @@ -12,11 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from fabric import api as fabric_api - from solar.core.log import log from solar.core.transports.base import Executor from solar.core.transports.base import SyncTransport +from solar.utils import execute # XXX: # currently we don't support key verification or acceptation @@ -55,9 +54,8 @@ class RsyncSyncTransport(SyncTransport): _from=_from, _to=_to) - rsync_executor = lambda transport: fabric_api.local( - rsync_cmd - ) + rsync_executor = lambda transport: execute( + rsync_cmd, shell=True) log.debug("RSYNC CMD: %r" % rsync_cmd) diff --git a/solar/core/transports/ssh_raw.py b/solar/core/transports/ssh_raw.py index 2d73f5c0..31a66a7e 100644 --- a/solar/core/transports/ssh_raw.py +++ b/solar/core/transports/ssh_raw.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from fabric import api as fabric_api from solar.core.log import log from solar.core.transports.base import RunTransport +from solar.utils import execute class _RawSSHTransport(object): @@ -68,5 +68,6 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport): ssh_cmd += (self._ssh_command_host(settings), remote_cmd) log.debug("RAW SSH CMD: %r", ssh_cmd) + # TODO convert it to SolarRunResult - return fabric_api.local(' '.join(ssh_cmd)) + return execute(' '.join(ssh_cmd), shell=True) diff --git a/solar/utils.py b/solar/utils.py index 813b8488..5f30214a 100644 --- a/solar/utils.py +++ b/solar/utils.py @@ -43,6 +43,17 @@ def communicate(command, data): stderr=subprocess.PIPE) return popen.communicate(input=data)[0] + +def execute(command, shell=False): + popen = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=shell) + out, err = popen.communicate() + return popen.returncode, out, err + + # Configure jinja2 filters jinja_env_with_filters = Environment() jinja_env_with_filters.filters['to_json'] = to_json From 8ad488a866f5fbce364bad98935d58028662fcf2 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 14:46:34 +0200 Subject: [PATCH 14/15] Add doc about StrictHostKeyChecking --- doc/source/transports.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/source/transports.rst b/doc/source/transports.rst index 950a681e..f5ead257 100644 --- a/doc/source/transports.rst +++ b/doc/source/transports.rst @@ -31,6 +31,16 @@ Currently there are following sync transports available: * solar_agent * torrent +Ssh host key checking +--------------------- +Solar wont disable strict host key checking by default, so before working +with solar ensure that strict host key checking is disabled, or all target hosts added to .ssh/known_hosts file. + +Example of .ssh/config :: + + Host 10.0.0.* + StrictHostKeyChecking no + Run transport ------------- From a73746a621c09902a52865c3577e3920e5fe0b75 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 10 Dec 2015 18:38:35 +0200 Subject: [PATCH 15/15] Raise sync error in Executor --- solar/core/handlers/ansible_template.py | 3 +-- solar/core/handlers/base.py | 9 --------- solar/core/handlers/naive_sync.py | 3 +-- solar/core/handlers/puppet.py | 3 +-- solar/core/handlers/shell.py | 4 +--- solar/core/transports/base.py | 18 ++++++++++++------ solar/core/transports/bat.py | 6 +----- 7 files changed, 17 insertions(+), 29 deletions(-) diff --git a/solar/core/handlers/ansible_template.py b/solar/core/handlers/ansible_template.py index e0990e89..85170a3a 100644 --- a/solar/core/handlers/ansible_template.py +++ b/solar/core/handlers/ansible_template.py @@ -39,8 +39,7 @@ class AnsibleTemplate(TempFileHandler): self._copy_templates_and_scripts(resource, action_name) self.transport_sync.copy(resource, self.dst, '/tmp') self.transport_sync.copy(resource, '/vagrant/library', '/tmp') - sync_results = self.transport_sync.sync_all() - self.verify_sync_results(sync_results) + self.transport_sync.sync_all() # remote paths are not nested inside solar_local remote_playbook_file = playbook_file.replace( diff --git a/solar/core/handlers/base.py b/solar/core/handlers/base.py index 86d6cb23..e90818ed 100644 --- a/solar/core/handlers/base.py +++ b/solar/core/handlers/base.py @@ -43,15 +43,6 @@ class BaseHandler(object): self.transport_sync.bind_with(self.transport_run) self.transport_run.bind_with(self.transport_sync) - def verify_sync_results(self, results): - for result in results: - if isinstance(result, tuple) and len(result) == 3: - # TODO Include file information in result - rc, out, err = result - log.debug('RC %s OUT %s ERR %s', rc, out, err) - if rc: - raise errors.SolarError(err) - def verify_run_result(self, cmd, result): rc, out, err = result log.debug('CMD %r RC %s OUT %s ERR %s', cmd, rc, out, err) diff --git a/solar/core/handlers/naive_sync.py b/solar/core/handlers/naive_sync.py index 82f87095..c06b7236 100644 --- a/solar/core/handlers/naive_sync.py +++ b/solar/core/handlers/naive_sync.py @@ -27,5 +27,4 @@ class NaiveSync(BaseHandler): # to understand where src comes from for item in args['sources']: self.transport_sync.copy(resource, item['src'], item['dst']) - results = self.transport_sync.sync_all() - self.verify_sync_results(results) + self.transport_sync.sync_all() diff --git a/solar/core/handlers/puppet.py b/solar/core/handlers/puppet.py index baf7fade..524130c9 100644 --- a/solar/core/handlers/puppet.py +++ b/solar/core/handlers/puppet.py @@ -37,8 +37,7 @@ class Puppet(TempFileHandler): self.prepare_templates_and_scripts(resource, action_file, '') self.transport_sync.copy(resource, action_file, action_file_name) - sync_results = self.transport_sync.sync_all() - self.verify_sync_results(sync_results) + self.transport_sync.sync_all() cmd_args = ['puppet', 'apply', '-vd', action_file_name, diff --git a/solar/core/handlers/shell.py b/solar/core/handlers/shell.py index d2c5b023..a028d3f0 100644 --- a/solar/core/handlers/shell.py +++ b/solar/core/handlers/shell.py @@ -33,9 +33,7 @@ class Shell(TempFileHandler): self._copy_templates_and_scripts(resource, action_name) self.transport_sync.copy(resource, self.dst, '/tmp') - sync_results = self.transport_sync.sync_all() - # TODO Include file information in result - self.verify_sync_results(sync_results) + self.transport_sync.sync_all() rst = self.transport_run.run( resource, diff --git a/solar/core/transports/base.py b/solar/core/transports/base.py index c867d5fd..1dbca863 100644 --- a/solar/core/transports/base.py +++ b/solar/core/transports/base.py @@ -12,6 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. +from solar.core.log import log +from solar import errors + class Executor(object): @@ -38,7 +41,13 @@ class Executor(object): def run(self, transport): if self.valid: - return self._executor(transport) + result = self._executor(transport) + if isinstance(result, tuple) and len(result) == 3: + # TODO Include file information in result + rc, out, err = result + log.debug('RC %s OUT %s ERR %s', rc, out, err) + if rc: + raise errors.SolarError(err) class SolarRunResultWrp(object): @@ -131,10 +140,8 @@ class SyncTransport(SolarTransport): self.preprocess(executor) def run_all(self): - rst = [] for executor in self.executors: - rst.append(executor.run(self)) - return rst + executor.run(self) def sync_all(self): """Syncs all @@ -144,9 +151,8 @@ class SyncTransport(SolarTransport): Could be someday changed to parallel thing. """ self.preprocess_all() - rst = self.run_all() + self.run_all() self.executors = [] # clear after all - return rst class RunTransport(SolarTransport): diff --git a/solar/core/transports/bat.py b/solar/core/transports/bat.py index 32c66337..5843ed88 100644 --- a/solar/core/transports/bat.py +++ b/solar/core/transports/bat.py @@ -60,12 +60,8 @@ class OnAll(object): def __get__(self, obj, objtype): def _inner(*args, **kwargs): - results = [] for transport in obj._used_transports: - rst = getattr(transport, self._target)(*args, **kwargs) - if rst: - results.extend(rst) - return results + getattr(transport, self._target)(*args, **kwargs) return _inner