From 4955d1ada72782fee60fc623e6fa76a7a29588a2 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 18 Aug 2015 13:34:35 -0400 Subject: [PATCH] [corey.bryant,r=trivial] Sync charm-helpers to pick up Liberty support. --- hooks/charmhelpers/cli/__init__.py | 6 +- hooks/charmhelpers/cli/commands.py | 8 +- hooks/charmhelpers/cli/hookenv.py | 23 ++++ .../contrib/openstack/amulet/deployment.py | 4 +- hooks/charmhelpers/contrib/openstack/utils.py | 65 ++++++++--- .../contrib/storage/linux/utils.py | 5 +- hooks/charmhelpers/core/hookenv.py | 21 +--- hooks/charmhelpers/core/host.py | 25 ++++- hooks/charmhelpers/core/services/helpers.py | 20 +++- hooks/charmhelpers/fetch/__init__.py | 8 ++ tests/charmhelpers/contrib/amulet/utils.py | 105 ++++++++++++++---- .../contrib/openstack/amulet/deployment.py | 4 +- 12 files changed, 220 insertions(+), 74 deletions(-) create mode 100644 hooks/charmhelpers/cli/hookenv.py diff --git a/hooks/charmhelpers/cli/__init__.py b/hooks/charmhelpers/cli/__init__.py index 7118daf5..16d52cc4 100644 --- a/hooks/charmhelpers/cli/__init__.py +++ b/hooks/charmhelpers/cli/__init__.py @@ -152,15 +152,11 @@ class CommandLine(object): arguments = self.argument_parser.parse_args() argspec = inspect.getargspec(arguments.func) vargs = [] - kwargs = {} for arg in argspec.args: vargs.append(getattr(arguments, arg)) if argspec.varargs: vargs.extend(getattr(arguments, argspec.varargs)) - if argspec.keywords: - for kwarg in argspec.keywords.items(): - kwargs[kwarg] = getattr(arguments, kwarg) - output = arguments.func(*vargs, **kwargs) + output = arguments.func(*vargs) if getattr(arguments.func, '_cli_test_command', False): self.exit_code = 0 if output else 1 output = '' diff --git a/hooks/charmhelpers/cli/commands.py b/hooks/charmhelpers/cli/commands.py index 443ff05d..7e91db00 100644 --- a/hooks/charmhelpers/cli/commands.py +++ b/hooks/charmhelpers/cli/commands.py @@ -26,7 +26,7 @@ from . import CommandLine # noqa """ Import the sub-modules which have decorated subcommands to register with chlp. """ -import host # noqa -import benchmark # noqa -import unitdata # noqa -from charmhelpers.core import hookenv # noqa +from . import host # noqa +from . import benchmark # noqa +from . import unitdata # noqa +from . import hookenv # noqa diff --git a/hooks/charmhelpers/cli/hookenv.py b/hooks/charmhelpers/cli/hookenv.py new file mode 100644 index 00000000..265c816e --- /dev/null +++ b/hooks/charmhelpers/cli/hookenv.py @@ -0,0 +1,23 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +from . import cmdline +from charmhelpers.core import hookenv + + +cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped) +cmdline.subcommand('service-name')(hookenv.service_name) +cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index b01e6cb8..07ee2ef1 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb'] + base_charms = ['mysql', 'mongodb', 'nrpe'] if self.series in ['precise', 'trusty']: base_series = self.series @@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): 'ceph-osd', 'ceph-radosgw'] # Most OpenStack subordinate charms do not expose an origin option # as that is controlled by the principle. - ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch'] + ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] if self.openstack: for svc in services: diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 4dd000c3..c9fd68f7 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -24,6 +24,7 @@ import subprocess import json import os import sys +import re import six import yaml @@ -69,7 +70,6 @@ CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' 'restricted main multiverse universe') - UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('oneiric', 'diablo'), ('precise', 'essex'), @@ -118,6 +118,34 @@ SWIFT_CODENAMES = OrderedDict([ ('2.3.0', 'liberty'), ]) +# >= Liberty version->codename mapping +PACKAGE_CODENAMES = { + 'nova-common': OrderedDict([ + ('12.0.0', 'liberty'), + ]), + 'neutron-common': OrderedDict([ + ('7.0.0', 'liberty'), + ]), + 'cinder-common': OrderedDict([ + ('7.0.0', 'liberty'), + ]), + 'keystone': OrderedDict([ + ('8.0.0', 'liberty'), + ]), + 'horizon-common': OrderedDict([ + ('8.0.0', 'liberty'), + ]), + 'ceilometer-common': OrderedDict([ + ('5.0.0', 'liberty'), + ]), + 'heat-common': OrderedDict([ + ('5.0.0', 'liberty'), + ]), + 'glance-common': OrderedDict([ + ('11.0.0', 'liberty'), + ]), +} + DEFAULT_LOOPBACK_SIZE = '5G' @@ -201,20 +229,29 @@ def get_os_codename_package(package, fatal=True): error_out(e) vers = apt.upstream_version(pkg.current_ver.ver_str) + match = re.match('^(\d)\.(\d)\.(\d)', vers) + if match: + vers = match.group(0) - try: - if 'swift' in pkg.name: - swift_vers = vers[:5] - if swift_vers not in SWIFT_CODENAMES: - # Deal with 1.10.0 upward - swift_vers = vers[:6] - return SWIFT_CODENAMES[swift_vers] - else: - vers = vers[:6] - return OPENSTACK_CODENAMES[vers] - except KeyError: - e = 'Could not determine OpenStack codename for version %s' % vers - error_out(e) + # >= Liberty independent project versions + if (package in PACKAGE_CODENAMES and + vers in PACKAGE_CODENAMES[package]): + return PACKAGE_CODENAMES[package][vers] + else: + # < Liberty co-ordinated project versions + try: + if 'swift' in pkg.name: + swift_vers = vers[:5] + if swift_vers not in SWIFT_CODENAMES: + # Deal with 1.10.0 upward + swift_vers = vers[:6] + return SWIFT_CODENAMES[swift_vers] + else: + vers = vers[:6] + return OPENSTACK_CODENAMES[vers] + except KeyError: + e = 'Could not determine OpenStack codename for version %s' % vers + error_out(e) def get_os_version_package(pkg, fatal=True): diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index e2769e49..1e57941a 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -43,9 +43,10 @@ def zap_disk(block_device): :param block_device: str: Full path of block device to clean. ''' + # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b # sometimes sgdisk exits non-zero; this is OK, dd will clean up - call(['sgdisk', '--zap-all', '--mbrtogpt', - '--clear', block_device]) + call(['sgdisk', '--zap-all', '--', block_device]) + call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device]) dev_end = check_output(['blockdev', '--getsz', block_device]).decode('UTF-8') gpt_end = int(dev_end.split()[0]) - 100 diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index 18860f59..a35d006b 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -34,23 +34,6 @@ import errno import tempfile from subprocess import CalledProcessError -try: - from charmhelpers.cli import cmdline -except ImportError as e: - # due to the anti-pattern of partially synching charmhelpers directly - # into charms, it's possible that charmhelpers.cli is not available; - # if that's the case, they don't really care about using the cli anyway, - # so mock it out - if str(e) == 'No module named cli': - class cmdline(object): - @classmethod - def subcommand(cls, *args, **kwargs): - def _wrap(func): - return func - return _wrap - else: - raise - import six if not six.PY3: from UserDict import UserDict @@ -91,6 +74,7 @@ def cached(func): res = func(*args, **kwargs) cache[key] = res return res + wrapper._wrapped = func return wrapper @@ -190,7 +174,6 @@ def relation_type(): return os.environ.get('JUJU_RELATION', None) -@cmdline.subcommand() @cached def relation_id(relation_name=None, service_or_unit=None): """The relation ID for the current or a specified relation""" @@ -216,13 +199,11 @@ def remote_unit(): return os.environ.get('JUJU_REMOTE_UNIT', None) -@cmdline.subcommand() def service_name(): """The name service group this unit belongs to""" return local_unit().split('/')[0] -@cmdline.subcommand() @cached def remote_service_name(relid=None): """The remote service name for a given relation-id (or the current relation)""" diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8ae8ef86..ec659eef 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -72,7 +72,7 @@ def service_pause(service_name, init_dir=None): stopped = service_stop(service_name) # XXX: Support systemd too override_path = os.path.join( - init_dir, '{}.conf.override'.format(service_name)) + init_dir, '{}.override'.format(service_name)) with open(override_path, 'w') as fh: fh.write("manual\n") return stopped @@ -86,7 +86,7 @@ def service_resume(service_name, init_dir=None): if init_dir is None: init_dir = "/etc/init" override_path = os.path.join( - init_dir, '{}.conf.override'.format(service_name)) + init_dir, '{}.override'.format(service_name)) if os.path.exists(override_path): os.unlink(override_path) started = service_start(service_name) @@ -148,6 +148,16 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False): return user_info +def user_exists(username): + """Check if a user exists""" + try: + pwd.getpwnam(username) + user_exists = True + except KeyError: + user_exists = False + return user_exists + + def add_group(group_name, system_group=False): """Add a group to the system""" try: @@ -280,6 +290,17 @@ def mounts(): return system_mounts +def fstab_mount(mountpoint): + """Mount filesystem using fstab""" + cmd_args = ['mount', mountpoint] + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error unmounting {}\n{}'.format(mountpoint, e.output)) + return False + return True + + def file_hash(path, hash_type='md5'): """ Generate a hash checksum of the contents of 'path' or None if not found. diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 8005c415..3f677833 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -16,7 +16,9 @@ import os import yaml + from charmhelpers.core import hookenv +from charmhelpers.core import host from charmhelpers.core import templating from charmhelpers.core.services.base import ManagerCallback @@ -240,27 +242,41 @@ class TemplateCallback(ManagerCallback): :param str source: The template source file, relative to `$CHARM_DIR/templates` + :param str target: The target to write the rendered template to :param str owner: The owner of the rendered file :param str group: The group of the rendered file :param int perms: The permissions of the rendered file - + :param partial on_change_action: functools partial to be executed when + rendered file changes """ def __init__(self, source, target, - owner='root', group='root', perms=0o444): + owner='root', group='root', perms=0o444, + on_change_action=None): self.source = source self.target = target self.owner = owner self.group = group self.perms = perms + self.on_change_action = on_change_action def __call__(self, manager, service_name, event_name): + pre_checksum = '' + if self.on_change_action and os.path.isfile(self.target): + pre_checksum = host.file_hash(self.target) service = manager.get_service(service_name) context = {} for ctx in service.get('required_data', []): context.update(ctx) templating.render(self.source, self.target, context, self.owner, self.group, self.perms) + if self.on_change_action: + if pre_checksum == host.file_hash(self.target): + hookenv.log( + 'No change detected: {}'.format(self.target), + hookenv.DEBUG) + else: + self.on_change_action() # Convenience aliases for templates diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 0a3bb969..cd0b783c 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -90,6 +90,14 @@ CLOUD_ARCHIVE_POCKETS = { 'kilo/proposed': 'trusty-proposed/kilo', 'trusty-kilo/proposed': 'trusty-proposed/kilo', 'trusty-proposed/kilo': 'trusty-proposed/kilo', + # Liberty + 'liberty': 'trusty-updates/liberty', + 'trusty-liberty': 'trusty-updates/liberty', + 'trusty-liberty/updates': 'trusty-updates/liberty', + 'trusty-updates/liberty': 'trusty-updates/liberty', + 'liberty/proposed': 'trusty-proposed/liberty', + 'trusty-liberty/proposed': 'trusty-proposed/liberty', + 'trusty-proposed/liberty': 'trusty-proposed/liberty', } # The order of this list is very important. Handlers should be listed in from diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 3de26afd..7816c934 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -14,17 +14,23 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -import amulet -import ConfigParser -import distro_info import io +import json import logging import os import re -import six +import subprocess import sys import time -import urlparse + +import amulet +import distro_info +import six +from six.moves import configparser +if six.PY3: + from urllib import parse as urlparse +else: + import urlparse class AmuletUtils(object): @@ -142,19 +148,23 @@ class AmuletUtils(object): for service_name in services_list: if (self.ubuntu_releases.index(release) >= systemd_switch or - service_name == "rabbitmq-server"): - # init is systemd + service_name in ['rabbitmq-server', 'apache2']): + # init is systemd (or regular sysv) cmd = 'sudo service {} status'.format(service_name) + output, code = sentry_unit.run(cmd) + service_running = code == 0 elif self.ubuntu_releases.index(release) < systemd_switch: # init is upstart cmd = 'sudo status {}'.format(service_name) + output, code = sentry_unit.run(cmd) + service_running = code == 0 and "start/running" in output - output, code = sentry_unit.run(cmd) self.log.debug('{} `{}` returned ' '{}'.format(sentry_unit.info['unit_name'], cmd, code)) - if code != 0: - return "command `{}` returned {}".format(cmd, str(code)) + if not service_running: + return u"command `{}` returned {} {}".format( + cmd, output, str(code)) return None def _get_config(self, unit, filename): @@ -164,7 +174,7 @@ class AmuletUtils(object): # NOTE(beisner): by default, ConfigParser does not handle options # with no value, such as the flags used in the mysql my.cnf file. # https://bugs.python.org/issue7005 - config = ConfigParser.ConfigParser(allow_no_value=True) + config = configparser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config @@ -450,15 +460,20 @@ class AmuletUtils(object): cmd, code, output)) return None - def get_process_id_list(self, sentry_unit, process_name): + def get_process_id_list(self, sentry_unit, process_name, + expect_success=True): """Get a list of process ID(s) from a single sentry juju unit for a single process name. - :param sentry_unit: Pointer to amulet sentry instance (juju unit) + :param sentry_unit: Amulet sentry instance (juju unit) :param process_name: Process name + :param expect_success: If False, expect the PID to be missing, + raise if it is present. :returns: List of process IDs """ - cmd = 'pidof {}'.format(process_name) + cmd = 'pidof -x {}'.format(process_name) + if not expect_success: + cmd += " || exit 0 && exit 1" output, code = sentry_unit.run(cmd) if code != 0: msg = ('{} `{}` returned {} ' @@ -467,14 +482,23 @@ class AmuletUtils(object): amulet.raise_status(amulet.FAIL, msg=msg) return str(output).split() - def get_unit_process_ids(self, unit_processes): + def get_unit_process_ids(self, unit_processes, expect_success=True): """Construct a dict containing unit sentries, process names, and - process IDs.""" + process IDs. + + :param unit_processes: A dictionary of Amulet sentry instance + to list of process names. + :param expect_success: if False expect the processes to not be + running, raise if they are. + :returns: Dictionary of Amulet sentry instance to dictionary + of process names to PIDs. + """ pid_dict = {} - for sentry_unit, process_list in unit_processes.iteritems(): + for sentry_unit, process_list in six.iteritems(unit_processes): pid_dict[sentry_unit] = {} for process in process_list: - pids = self.get_process_id_list(sentry_unit, process) + pids = self.get_process_id_list( + sentry_unit, process, expect_success=expect_success) pid_dict[sentry_unit].update({process: pids}) return pid_dict @@ -488,7 +512,7 @@ class AmuletUtils(object): return ('Unit count mismatch. expected, actual: {}, ' '{} '.format(len(expected), len(actual))) - for (e_sentry, e_proc_names) in expected.iteritems(): + for (e_sentry, e_proc_names) in six.iteritems(expected): e_sentry_name = e_sentry.info['unit_name'] if e_sentry in actual.keys(): a_proc_names = actual[e_sentry] @@ -507,11 +531,23 @@ class AmuletUtils(object): '{}'.format(e_proc_name, a_proc_name)) a_pids_length = len(a_pids) - if e_pids_length != a_pids_length: - return ('PID count mismatch. {} ({}) expected, actual: ' + fail_msg = ('PID count mismatch. {} ({}) expected, actual: ' '{}, {} ({})'.format(e_sentry_name, e_proc_name, e_pids_length, a_pids_length, a_pids)) + + # If expected is not bool, ensure PID quantities match + if not isinstance(e_pids_length, bool) and \ + a_pids_length != e_pids_length: + return fail_msg + # If expected is bool True, ensure 1 or more PIDs exist + elif isinstance(e_pids_length, bool) and \ + e_pids_length is True and a_pids_length < 1: + return fail_msg + # If expected is bool False, ensure 0 PIDs exist + elif isinstance(e_pids_length, bool) and \ + e_pids_length is False and a_pids_length != 0: + return fail_msg else: self.log.debug('PID check OK: {} {} {}: ' '{}'.format(e_sentry_name, e_proc_name, @@ -531,3 +567,30 @@ class AmuletUtils(object): return 'Dicts within list are not identical' return None + + def run_action(self, unit_sentry, action, + _check_output=subprocess.check_output): + """Run the named action on a given unit sentry. + + _check_output parameter is used for dependency injection. + + @return action_id. + """ + unit_id = unit_sentry.info["unit_name"] + command = ["juju", "action", "do", "--format=json", unit_id, action] + self.log.info("Running command: %s\n" % " ".join(command)) + output = _check_output(command, universal_newlines=True) + data = json.loads(output) + action_id = data[u'Action queued with id'] + return action_id + + def wait_on_action(self, action_id, _check_output=subprocess.check_output): + """Wait for a given action, returning if it completed or not. + + _check_output parameter is used for dependency injection. + """ + command = ["juju", "action", "fetch", "--format=json", "--wait=0", + action_id] + output = _check_output(command, universal_newlines=True) + data = json.loads(output) + return data.get(u"status") == "completed" diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index b01e6cb8..07ee2ef1 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -44,7 +44,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): Determine if the local branch being tested is derived from its stable or next (dev) branch, and based on this, use the corresonding stable or next branches for the other_services.""" - base_charms = ['mysql', 'mongodb'] + base_charms = ['mysql', 'mongodb', 'nrpe'] if self.series in ['precise', 'trusty']: base_series = self.series @@ -81,7 +81,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): 'ceph-osd', 'ceph-radosgw'] # Most OpenStack subordinate charms do not expose an origin option # as that is controlled by the principle. - ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch'] + ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] if self.openstack: for svc in services: