diff --git a/hooks/charmhelpers/contrib/charmsupport/nrpe.py b/hooks/charmhelpers/contrib/charmsupport/nrpe.py index 0fd0a9d8..9d961cfb 100644 --- a/hooks/charmhelpers/contrib/charmsupport/nrpe.py +++ b/hooks/charmhelpers/contrib/charmsupport/nrpe.py @@ -24,6 +24,8 @@ import subprocess import pwd import grp import os +import glob +import shutil import re import shlex import yaml @@ -161,7 +163,7 @@ define service {{ log('Check command not found: {}'.format(parts[0])) return '' - def write(self, nagios_context, hostname, nagios_servicegroups=None): + def write(self, nagios_context, hostname, nagios_servicegroups): nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( self.command) with open(nrpe_check_file, 'w') as nrpe_check_config: @@ -177,14 +179,11 @@ define service {{ nagios_servicegroups) def write_service_config(self, nagios_context, hostname, - nagios_servicegroups=None): + nagios_servicegroups): for f in os.listdir(NRPE.nagios_exportdir): if re.search('.*{}.cfg'.format(self.command), f): os.remove(os.path.join(NRPE.nagios_exportdir, f)) - if not nagios_servicegroups: - nagios_servicegroups = nagios_context - templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, @@ -211,10 +210,10 @@ class NRPE(object): super(NRPE, self).__init__() self.config = config() self.nagios_context = self.config['nagios_context'] - if 'nagios_servicegroups' in self.config: + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: self.nagios_servicegroups = self.config['nagios_servicegroups'] else: - self.nagios_servicegroups = 'juju' + self.nagios_servicegroups = self.nagios_context self.unit_name = local_unit().replace('/', '-') if hostname: self.hostname = hostname @@ -322,3 +321,38 @@ def add_init_service_checks(nrpe, services, unit_name): check_cmd='check_status_file.py -f ' '/var/lib/nagios/service-check-%s.txt' % svc, ) + + +def copy_nrpe_checks(): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', + 'charmhelpers', 'contrib', 'openstack', + 'files') + + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 9a2588b6..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -164,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py index 1e79a0ca..560e6a03 100644 --- a/hooks/charmhelpers/contrib/network/ufw.py +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -37,19 +37,22 @@ Examples: >>> ufw.enable() >>> ufw.service('4949', 'close') # munin """ - -__author__ = "Felipe Reyes " - import re import os import subprocess from charmhelpers.core import hookenv +__author__ = "Felipe Reyes " + class UFWError(Exception): pass +class UFWIPv6Error(UFWError): + pass + + def is_enabled(): """ Check if `ufw` is enabled @@ -66,10 +69,13 @@ def is_enabled(): return len(m) >= 1 -def is_ipv6_ok(): +def is_ipv6_ok(soft_fail=False): """ Check if IPv6 support is present and ip6tables functional + :param soft_fail: If set to True and IPv6 support is broken, then reports + that the host doesn't have IPv6 support, otherwise a + UFWIPv6Error exception is raised. :returns: True if IPv6 is working, False otherwise """ @@ -89,8 +95,11 @@ def is_ipv6_ok(): hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, level="WARN") # we are in a world where ip6tables isn't working - # so we inform that the machine doesn't have IPv6 - return False + if soft_fail: + # so we inform that the machine doesn't have IPv6 + return False + else: + raise UFWIPv6Error("IPv6 firewall support broken") else: # the module is present :) return True @@ -113,16 +122,19 @@ def disable_ipv6(): raise UFWError("Couldn't disable IPv6 support in ufw") -def enable(): +def enable(soft_fail=False): """ Enable ufw + :param soft_fail: If set to True silently disables IPv6 support in ufw, + otherwise a UFWIPv6Error exception is raised when IP6 + support is broken. :returns: True if ufw is successfully enabled """ if is_enabled(): return True - if not is_ipv6_ok(): + if not is_ipv6_ok(soft_fail): disable_ipv6() output = subprocess.check_output(['ufw', 'enable'], diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config) diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 9eabed73..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 26259a03..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index d848a120..8659516b 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = "Jorge Niedbalski " - from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import log @@ -29,6 +27,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +__author__ = "Jorge Niedbalski " + def parse_options(given, available): """Given a set of options, check if available""" diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index be7de248..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -17,11 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import io import os +__author__ = 'Jorge Niedbalski R. ' + class Fstab(io.FileIO): """This class extends file in order to implement a file reader/writer @@ -77,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -104,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py new file mode 100644 index 00000000..efc4402e --- /dev/null +++ b/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in ['y', 'yes', 'true', 't']: + return True + elif value in ['n', 'no', 'false', 'f']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 8e1b9eeb..21cc8ab2 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . -__author__ = 'Jorge Niedbalski R. ' - import yaml from subprocess import check_call @@ -29,6 +27,8 @@ from charmhelpers.core.hookenv import ( ERROR, ) +__author__ = 'Jorge Niedbalski R. ' + def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 01329ab7..3000134a 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -435,7 +435,7 @@ class HookData(object): os.path.join(charm_dir, 'revision')).read().strip() charm_rev = charm_rev or '0' revs = self.kv.get('charm_revisions', []) - if not charm_rev in revs: + if charm_rev not in revs: revs.append(charm_rev.strip() or '0') self.kv.set('charm_revisions', revs) diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index d25a0ddd..8dfce505 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -18,6 +18,16 @@ import os import hashlib import re +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + import six if six.PY3: from urllib.request import ( @@ -35,16 +45,6 @@ else: ) from urlparse import urlparse, urlunparse, parse_qs -from charmhelpers.fetch import ( - BaseFetchHandler, - UnhandledSource -) -from charmhelpers.payload.archive import ( - get_archive_handler, - extract, -) -from charmhelpers.core.host import mkdir, check_hash - def splituser(host): '''urllib.splituser(), but six's support of this seems broken''' diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 5376786b..93aae87b 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -32,7 +32,7 @@ except ImportError: apt_install("python-git") from git import Repo -from git.exc import GitCommandError +from git.exc import GitCommandError # noqa E402 class GitUrlFetchHandler(BaseFetchHandler): diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 3464b873..65219d33 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -169,8 +169,13 @@ class AmuletUtils(object): cmd = 'pgrep -o -f {}'.format(service) else: cmd = 'pgrep -o {}'.format(service) - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) - return self._get_dir_mtime(sentry_unit, proc_dir) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) def service_restarted(self, sentry_unit, service, filename, pgrep_full=False, sleep_time=20): @@ -187,6 +192,121 @@ class AmuletUtils(object): else: return False + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + def relation_error(self, name, data): return 'unexpected relation data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index c50d3ec6..0cfeaa4c 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -71,16 +71,19 @@ class OpenStackAmuletDeployment(AmuletDeployment): services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] if self.openstack: for svc in services: - if svc['name'] not in use_source: + if svc['name'] not in use_source + ignore: config = {'openstack-origin': self.openstack} self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc['name'] in use_source: + if svc['name'] in use_source and svc['name'] not in ignore: config = {'source': self.source} self.d.configure(svc['name'], config)