From 73f13f0259309129addd024661346277c8abf016 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 9 Jul 2015 13:58:15 +0200 Subject: [PATCH 1/7] resource action dry-run added with possibility to modify fabric return values --- example-puppet.py | 1 - solar/solar/cli.py | 37 ++++++++++++++++++++++++++++- solar/solar/core/handlers/puppet.py | 1 + 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/example-puppet.py b/example-puppet.py index 93f3fa77..6b72b9c8 100644 --- a/example-puppet.py +++ b/example-puppet.py @@ -4,7 +4,6 @@ import time from solar.core import actions from solar.core import resource -from solar.core.provider import GitProvider from solar.core import signals from solar.core import validation from solar.core import virtual_resource as vr diff --git a/solar/solar/cli.py b/solar/solar/cli.py index aacdcd44..2dc11949 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -20,6 +20,7 @@ On create "golden" resource should be moved to special place import click from fabric import api as fabric_api import json +from hashlib import md5 import networkx as nx import os import pprint @@ -283,13 +284,47 @@ def init_cli_resource(): @resource.command() @click.argument('resource_name') @click.argument('action_name') - def action(action_name, resource_name): + @click.option('-d', '--dry-run', default=False, is_flag=True) + @click.option('-m', '--mapping', default='{}') + def action(mapping, dry_run, action_name, resource_name): + from fabric import api as fabric_api + from fabric.contrib import project as fabric_project + import mock + + if dry_run: + mapping = json.loads(mapping) + + executed = [] + + def compute_hash(key): + return md5(str(key)).hexdigest() + + def dry_run_executor(command_name): + def wrapper(*args, **kwargs): + key = (command_name, args, kwargs) + + executed.append(key) + + return mapping.get(compute_hash(key), '') + + return wrapper + + fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) + fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) + click.echo( 'action {} for resource {}'.format(action_name, resource_name) ) r = sresource.load(resource_name) actions.resource_action(r, action_name) + print 'executed:' + for key in executed: + click.echo('{}: {}'.format( + click.style(compute_hash(key), fg='green'), + str(key) + )) + @resource.command() def clear_all(): click.echo('Clearing all resources') diff --git a/solar/solar/core/handlers/puppet.py b/solar/solar/core/handlers/puppet.py index 7599b6bb..9b323b98 100644 --- a/solar/solar/core/handlers/puppet.py +++ b/solar/solar/core/handlers/puppet.py @@ -62,6 +62,7 @@ class LibrarianPuppet(ResourceSSHMixin): self.resource, 'sudo', 'cat', '/tmp/puppet-modules/Puppetfile' ) + log.debug('Puppetlabs file is: \n%s\n', puppetlabs) git = self.resource.args['git'].value From a76c404c7361fca61eb17beb383dd962a55ac8b0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 9 Jul 2015 14:03:51 +0200 Subject: [PATCH 2/7] Resource action dry run: hash also call number This is to get unique hashes for same commands --- solar/solar/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 2dc11949..244c8bfc 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -301,7 +301,7 @@ def init_cli_resource(): def dry_run_executor(command_name): def wrapper(*args, **kwargs): - key = (command_name, args, kwargs) + key = (len(executed), command_name, args, kwargs) executed.append(key) From 779e5d9f4a0473f25df577fc76e79b02a6638ede Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 9 Jul 2015 14:28:18 +0200 Subject: [PATCH 3/7] Separate DryRunExecutor class, used also in 'solar run' command --- solar/solar/cli.py | 85 +++++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 244c8bfc..8da87e1b 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -48,6 +48,35 @@ from solar.extensions.modules.discovery import Discovery db = get_db() +class DryRunExecutor(object): + def __init__(self, mapping=None): + from fabric import api as fabric_api + from fabric.contrib import project as fabric_project + import mock + + self.executed = [] + + mapping = mapping or {} + + def dry_run_executor(command_name): + def wrapper(*args, **kwargs): + key = (len(self.executed), command_name, args, kwargs) + + self.executed.append(key) + + return mapping.get(self.compute_hash(key), '') + + return wrapper + + # Add your own mocks here, IO, whatever + fabric_api.local = mock.Mock(side_effect=dry_run_executor('LOCAL RUN')) + fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) + fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) + + def compute_hash(self, key): + return md5(str(key)).hexdigest() + + # HELPERS def format_resource_input(resource_name, resource_input_name): return '{}::{}'.format( @@ -132,10 +161,15 @@ def init_actions(): @main.command() @click.option('-t', '--tags') @click.option('-a', '--action') - def run(action, tags): + @click.option('-d', '--dry-run', default=False, is_flag=True) + @click.option('-m', '--dry-run-mapping', default='{}') + def run(dry_run_mapping, dry_run, action, tags): from solar.core import actions from solar.core import resource + if dry_run: + dry_run_executor = DryRunExecutor(mapping=json.loads(dry_run_mapping)) + resources = filter( lambda r: Expression(tags, r.get('tags', [])).evaluate(), db.get_list('resource')) @@ -144,6 +178,14 @@ def init_actions(): resource_obj = sresource.load(resource['id']) actions.resource_action(resource_obj, action) + if dry_run: + click.echo('EXECUTED:') + for key in dry_run_executor.executed: + click.echo('{}: {}'.format( + click.style(dry_run_executor.compute_hash(key), fg='green'), + str(key) + )) + def init_changes(): @main.group() @@ -285,32 +327,10 @@ def init_cli_resource(): @click.argument('resource_name') @click.argument('action_name') @click.option('-d', '--dry-run', default=False, is_flag=True) - @click.option('-m', '--mapping', default='{}') - def action(mapping, dry_run, action_name, resource_name): - from fabric import api as fabric_api - from fabric.contrib import project as fabric_project - import mock - + @click.option('-m', '--dry-run-mapping', default='{}') + def action(dry_run_mapping, dry_run, action_name, resource_name): if dry_run: - mapping = json.loads(mapping) - - executed = [] - - def compute_hash(key): - return md5(str(key)).hexdigest() - - def dry_run_executor(command_name): - def wrapper(*args, **kwargs): - key = (len(executed), command_name, args, kwargs) - - executed.append(key) - - return mapping.get(compute_hash(key), '') - - return wrapper - - fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) - fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) + dry_run_executor = DryRunExecutor(mapping=json.loads(dry_run_mapping)) click.echo( 'action {} for resource {}'.format(action_name, resource_name) @@ -318,12 +338,13 @@ def init_cli_resource(): r = sresource.load(resource_name) actions.resource_action(r, action_name) - print 'executed:' - for key in executed: - click.echo('{}: {}'.format( - click.style(compute_hash(key), fg='green'), - str(key) - )) + if dry_run: + click.echo('EXECUTED:') + for key in dry_run_executor.executed: + click.echo('{}: {}'.format( + click.style(dry_run_executor.compute_hash(key), fg='green'), + str(key) + )) @resource.command() def clear_all(): From 8e0cb6221e05e917baa1815f5df9c73c3dbfd969 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 15 Jul 2015 09:31:55 +0200 Subject: [PATCH 4/7] Dry run: hash startswith mapping, added possibility to read from file --- README.md | 37 +++++++++++++++++++++++++++++++++++++ main.yml | 1 + solar/solar/cli/main.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 722a64be..c0d711cd 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,43 @@ from x import resource all_resources = resource.load_all('rs') ``` +## Dry run + +Solar CLI has possibility to show dry run of actions to be performed. +To see what will happen when you run Puppet action, for example, try this: + +``` +solar resource action keystone_puppet run -d +``` + +This should print out something like this: + +``` +EXECUTED: +73c6cb1cf7f6cdd38d04dd2d0a0729f8: (0, 'SSH RUN', ('sudo cat /tmp/puppet-modules/Puppetfile',), {}) +3dd4d7773ce74187d5108ace0717ef29: (1, 'SSH SUDO', ('mv "1038cb062449340bdc4832138dca18cba75caaf8" "/tmp/puppet-modules/Puppetfile"',), {}) +ae5ad2455fe2b02ba46b4b7727eff01a: (2, 'SSH RUN', ('sudo librarian-puppet install',), {}) +208764fa257ed3159d1788f73c755f44: (3, 'SSH SUDO', ('puppet apply -vd /tmp/action.pp',), {}) +``` + +By default every mocked command returns an empty string. If you want it to return +something else (to check how would dry run behave in different situation) you provide +a mapping (in JSON format), something along the lines of: + +``` +solar resource action keystone_puppet run -d -m "{\"73c\": \"mod 'openstack-keystone'\n\"}" +``` + +The above means the return string of first command (with hash `73c6c...`) will be +as specified in the mapping. Notice that in mapping you don't have to specify the +whole hash, just it's unique beginning. Also, you don't have to specify the whole +return string in mapping. Dry run executor can read file and return it's contents +instead, just use the `>` operator when specifying hash: + +``` +solar resource action keystone_puppet run -d -m "{\"73c>\": \"./Puppetlabs-file\"}" +``` + ## CLI You can do the above from the command-line client: diff --git a/main.yml b/main.yml index 14158e8f..ef600c76 100644 --- a/main.yml +++ b/main.yml @@ -4,6 +4,7 @@ sudo: yes tasks: - apt: name=python-pip state=absent + - apt: name=python-six state=absent - shell: easy_install pip - shell: pip install -U pip - shell: pip install -U setuptools diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index a705ceff..5a26f7be 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -56,9 +56,11 @@ class DryRunExecutor(object): from fabric.contrib import project as fabric_project import mock + from solar.core.handlers import puppet + self.executed = [] - mapping = mapping or {} + self.mapping = mapping or {} def dry_run_executor(command_name): def wrapper(*args, **kwargs): @@ -66,18 +68,41 @@ class DryRunExecutor(object): self.executed.append(key) - return mapping.get(self.compute_hash(key), '') + return self.find_hash(self.compute_hash(key)) return wrapper # Add your own mocks here, IO, whatever fabric_api.local = mock.Mock(side_effect=dry_run_executor('LOCAL RUN')) + fabric_api.put = mock.Mock(side_effect=dry_run_executor('PUT')) fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) + fabric_api.sudo = mock.Mock(side_effect=dry_run_executor('SSH SUDO')) fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) def compute_hash(self, key): return md5(str(key)).hexdigest() + def find_hash(self, hash): + stripped_hashes = {k.replace('>', ''): k for k in self.mapping} + + hashes = [k for k in stripped_hashes if hash.startswith(k)] + + if len(hashes) == 0: + #raise Exception('Hash {} not found'.format(hash)) + return '' + elif len(hashes) > 1: + raise Exception('Hash {} not unique in {}'.format( + hash, hashes + )) + + hash = stripped_hashes[hashes[0]] + + if hash.endswith('>'): + with open(self.mapping[hash]) as f: + return f.read() + + return self.mapping[hash] + # HELPERS def format_resource_input(resource_name, resource_input_name): From d01efc93b085002a36e37fece03c25ee7f7c7265 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Wed, 5 Aug 2015 08:25:01 +0200 Subject: [PATCH 5/7] CLI: move DryRunExecutors to separate module --- solar/solar/cli/executors.py | 55 ++++++++++++++++++++++++++++++++ solar/solar/cli/main.py | 61 ++---------------------------------- 2 files changed, 58 insertions(+), 58 deletions(-) create mode 100644 solar/solar/cli/executors.py diff --git a/solar/solar/cli/executors.py b/solar/solar/cli/executors.py new file mode 100644 index 00000000..e67c30a1 --- /dev/null +++ b/solar/solar/cli/executors.py @@ -0,0 +1,55 @@ +from hashlib import md5 + + +class DryRunExecutor(object): + def __init__(self, mapping=None): + from fabric import api as fabric_api + from fabric.contrib import project as fabric_project + import mock + + from solar.core.handlers import puppet + + self.executed = [] + + self.mapping = mapping or {} + + def dry_run_executor(command_name): + def wrapper(*args, **kwargs): + key = (len(self.executed), command_name, args, kwargs) + + self.executed.append(key) + + return self.find_hash(self.compute_hash(key)) + + return wrapper + + # Add your own mocks here, IO, whatever + fabric_api.local = mock.Mock(side_effect=dry_run_executor('LOCAL RUN')) + fabric_api.put = mock.Mock(side_effect=dry_run_executor('PUT')) + fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) + fabric_api.sudo = mock.Mock(side_effect=dry_run_executor('SSH SUDO')) + fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) + + def compute_hash(self, key): + return md5(str(key)).hexdigest() + + def find_hash(self, hash): + stripped_hashes = {k.replace('>', ''): k for k in self.mapping} + + hashes = [k for k in stripped_hashes if hash.startswith(k)] + + if len(hashes) == 0: + #raise Exception('Hash {} not found'.format(hash)) + return '' + elif len(hashes) > 1: + raise Exception('Hash {} not unique in {}'.format( + hash, hashes + )) + + hash = stripped_hashes[hashes[0]] + + if hash.endswith('>'): + with open(self.mapping[hash]) as f: + return f.read() + + return self.mapping[hash] diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index 261ffb68..db525577 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -20,7 +20,6 @@ On create "golden" resource should be moved to special place import click from fabric import api as fabric_api import json -from hashlib import md5 import networkx as nx import os import pprint @@ -34,12 +33,12 @@ from solar.core import resource as sresource from solar.core.resource import assign_resources_to_nodes from solar.core import signals from solar.core.tags_set_parser import Expression -from solar.core import testing from solar.core.resource import virtual_resource as vr from solar.interfaces.db import get_db from solar import errors from solar.core.log import log +from solar.cli import executors from solar.cli.orch import orchestration from solar.cli.system_log import changes @@ -51,60 +50,6 @@ from solar.extensions.modules.discovery import Discovery db = get_db() -class DryRunExecutor(object): - def __init__(self, mapping=None): - from fabric import api as fabric_api - from fabric.contrib import project as fabric_project - import mock - - from solar.core.handlers import puppet - - self.executed = [] - - self.mapping = mapping or {} - - def dry_run_executor(command_name): - def wrapper(*args, **kwargs): - key = (len(self.executed), command_name, args, kwargs) - - self.executed.append(key) - - return self.find_hash(self.compute_hash(key)) - - return wrapper - - # Add your own mocks here, IO, whatever - fabric_api.local = mock.Mock(side_effect=dry_run_executor('LOCAL RUN')) - fabric_api.put = mock.Mock(side_effect=dry_run_executor('PUT')) - fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) - fabric_api.sudo = mock.Mock(side_effect=dry_run_executor('SSH SUDO')) - fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) - - def compute_hash(self, key): - return md5(str(key)).hexdigest() - - def find_hash(self, hash): - stripped_hashes = {k.replace('>', ''): k for k in self.mapping} - - hashes = [k for k in stripped_hashes if hash.startswith(k)] - - if len(hashes) == 0: - #raise Exception('Hash {} not found'.format(hash)) - return '' - elif len(hashes) > 1: - raise Exception('Hash {} not unique in {}'.format( - hash, hashes - )) - - hash = stripped_hashes[hashes[0]] - - if hash.endswith('>'): - with open(self.mapping[hash]) as f: - return f.read() - - return self.mapping[hash] - - # HELPERS def format_resource_input(resource_name, resource_input_name): return '{}::{}'.format( @@ -196,7 +141,7 @@ def init_actions(): from solar.core import resource if dry_run: - dry_run_executor = DryRunExecutor(mapping=json.loads(dry_run_mapping)) + dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) resources = filter( lambda r: Expression(tags, r.get('tags', [])).evaluate(), @@ -310,7 +255,7 @@ def init_cli_resource(): @click.option('-m', '--dry-run-mapping', default='{}') def action(dry_run_mapping, dry_run, action, resource): if dry_run: - dry_run_executor = DryRunExecutor(mapping=json.loads(dry_run_mapping)) + dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) click.echo( 'action {} for resource {}'.format(action, resource) From e858155ee7a822f331e9e29689d6619055c3fa96 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Wed, 5 Aug 2015 16:43:35 +0200 Subject: [PATCH 6/7] Fix undef default Signed-off-by: Bogdan Dobrelya --- resources/cinder_glance_puppet/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/cinder_glance_puppet/meta.yaml b/resources/cinder_glance_puppet/meta.yaml index 8d6a52b3..80b22525 100644 --- a/resources/cinder_glance_puppet/meta.yaml +++ b/resources/cinder_glance_puppet/meta.yaml @@ -17,7 +17,7 @@ input: value: false glance_request_timeout: schema: str - value: '' + value: git: schema: {repository: str!, branch: str!} From 3be289c49123f464ded777bf8692fd4ab3230e38 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Thu, 6 Aug 2015 10:58:09 +0200 Subject: [PATCH 7/7] Fix resource.py when value is not specified in meta.yaml --- solar/solar/core/resource/resource.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 1b4955ed..4a18927d 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -58,9 +58,7 @@ class Resource(object): self.metadata = raw_resource - args = self.metadata['input'] - - return {k: v['value'] for k, v in args.items()} + return Resource.get_raw_resource_args(raw_resource) def set_args_from_dict(self, new_args): args = self.args_dict() @@ -167,10 +165,14 @@ class Resource(object): else: raise Exception('Uuups, action is not available') + @staticmethod + def get_raw_resource_args(raw_resource): + return {k: v.get('value') for k, v in raw_resource['input'].items()} + def wrap_resource(raw_resource): name = raw_resource['id'] - args = {k: v['value'] for k, v in raw_resource['input'].items()} + args = Resource.get_raw_resource_args(raw_resource) tags = raw_resource.get('tags', []) virtual_resource = raw_resource.get('virtual_resource', [])