diff --git a/.gitignore b/.gitignore index a16ea07e..25d8aecb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin .tox tags *.sw[nop] +*.pyc diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 998f00c1..4efe7993 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -456,3 +456,18 @@ def get_hostname(address, fqdn=True): return result else: return result.split('.')[0] + + +def port_has_listener(address, port): + """ + Returns True if the address:port is open and being listened to, + else False. + + @param address: an IP address or hostname + @param port: integer port + + Note calls 'zc' via a subprocess shell + """ + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not(bool(result)) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index cbaad10d..d2ede320 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -121,7 +121,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Charms which should use the source config option use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', - 'ceph-osd', 'ceph-radosgw'] + 'ceph-osd', 'ceph-radosgw', 'ceph-mon'] # Charms which can not use openstack-origin, ie. many subordinates no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index e930bf37..a8c6ab0c 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -90,6 +90,12 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.core.unitdata import kv +try: + import psutil +except ImportError: + apt_install('python-psutil', fatal=True) + import psutil + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -404,6 +410,7 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' + api_version = rdata.get('api_version') or '2.0' ctxt.update({'service_port': rdata.get('service_port'), 'service_host': serv_host, 'auth_host': auth_host, @@ -412,7 +419,8 @@ class IdentityServiceContext(OSContextGenerator): 'admin_user': rdata.get('service_username'), 'admin_password': rdata.get('service_password'), 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol}) + 'auth_protocol': auth_protocol, + 'api_version': api_version}) if self.context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse @@ -1258,13 +1266,11 @@ class WorkerConfigContext(OSContextGenerator): @property def num_cpus(self): - try: - from psutil import NUM_CPUS - except ImportError: - apt_install('python-psutil', fatal=True) - from psutil import NUM_CPUS - - return NUM_CPUS + # NOTE: use cpu_count if present (16.04 support) + if hasattr(psutil, 'cpu_count'): + return psutil.cpu_count() + else: + return psutil.NUM_CPUS def __call__(self): multiplier = config('worker-multiplier') or 0 @@ -1467,6 +1473,8 @@ class NetworkServiceContext(OSContextGenerator): rdata.get('service_protocol') or 'http', 'auth_protocol': rdata.get('auth_protocol') or 'http', + 'api_version': + rdata.get('api_version') or '2.0', } if self.context_complete(ctxt): return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 9a8e0efc..d057ea6e 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -237,10 +237,16 @@ def neutron_plugins(): plugins['midonet']['driver'] = ( 'neutron.plugins.midonet.plugin.MidonetPluginV2') if release >= 'liberty': - midonet_origin = config('midonet-origin') - if midonet_origin is not None and midonet_origin[4:5] == '1': - plugins['midonet']['driver'] = ( - 'midonet.neutron.plugin_v1.MidonetPluginV2') + plugins['midonet']['driver'] = ( + 'midonet.neutron.plugin_v1.MidonetPluginV2') + plugins['midonet']['server_packages'].remove( + 'python-neutron-plugin-midonet') + plugins['midonet']['server_packages'].append( + 'python-networking-midonet') + plugins['plumgrid']['driver'] = ( + 'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2') + plugins['plumgrid']['server_packages'].remove( + 'neutron-plugin-plumgrid') return plugins diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend index ce28fa3f..6a923804 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf index ce28fa3f..6a923804 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -6,6 +6,8 @@ Listen {{ ext_port }} ServerName {{ endpoint }} SSLEngine on + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken index 2a37edd5..0b6da25c 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -1,4 +1,14 @@ {% if auth_host -%} +{% if api_version == '3' -%} +[keystone_authtoken] +auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +project_domain_name = default +user_domain_name = default +auth_plugin = password +{% else -%} [keystone_authtoken] identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} @@ -7,3 +17,4 @@ admin_user = {{ admin_user }} admin_password = {{ admin_password }} signing_dir = {{ signing_dir }} {% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 2ed7955c..80dd2e0d 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,8 +23,10 @@ import json import os import sys import re +import itertools import six +import tempfile import traceback import uuid import yaml @@ -41,6 +43,7 @@ from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, + DEBUG, INFO, related_units, relation_ids, @@ -58,6 +61,7 @@ from charmhelpers.contrib.storage.linux.lvm import ( from charmhelpers.contrib.network.ip import ( get_ipv6_addr, is_ipv6, + port_has_listener, ) from charmhelpers.contrib.python.packages import ( @@ -65,7 +69,7 @@ from charmhelpers.contrib.python.packages import ( pip_install, ) -from charmhelpers.core.host import lsb_release, mounts, umount +from charmhelpers.core.host import lsb_release, mounts, umount, service_running from charmhelpers.fetch import apt_install, apt_cache, install_remote from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -347,12 +351,42 @@ def os_release(package, base='essex'): def import_key(keyid): - cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ - "--recv-keys %s" % keyid - try: - subprocess.check_call(cmd.split(' ')) - except subprocess.CalledProcessError: - error_out("Error importing repo key %s" % keyid) + key = keyid.strip() + if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and + key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): + juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG) + juju_log("Importing ASCII Armor PGP key", level=DEBUG) + with tempfile.NamedTemporaryFile() as keyfile: + with open(keyfile.name, 'w') as fd: + fd.write(key) + fd.write("\n") + + cmd = ['apt-key', 'add', keyfile.name] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error_out("Error importing PGP key '%s'" % key) + else: + juju_log("PGP key found (looks like Radix64 format)", level=DEBUG) + juju_log("Importing PGP key from keyserver", level=DEBUG) + cmd = ['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error_out("Error importing PGP key '%s'" % key) + + +def get_source_and_pgp_key(input): + """Look for a pgp key ID or ascii-armor key in the given input.""" + index = input.strip() + index = input.rfind('|') + if index < 0: + return input, None + + key = input[index + 1:].strip('|') + source = input[:index] + return source, key def configure_installation_source(rel): @@ -364,16 +398,16 @@ def configure_installation_source(rel): with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: f.write(DISTRO_PROPOSED % ubuntu_rel) elif rel[:4] == "ppa:": - src = rel + src, key = get_source_and_pgp_key(rel) + if key: + import_key(key) + subprocess.check_call(["add-apt-repository", "-y", src]) elif rel[:3] == "deb": - l = len(rel.split('|')) - if l == 2: - src, key = rel.split('|') - juju_log("Importing PPA key from keyserver for %s" % src) + src, key = get_source_and_pgp_key(rel) + if key: import_key(key) - elif l == 1: - src = rel + with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: f.write(src) elif rel[:6] == 'cloud:': @@ -828,13 +862,23 @@ def os_workload_status(configs, required_interfaces, charm_func=None): return wrap -def set_os_workload_status(configs, required_interfaces, charm_func=None): +def set_os_workload_status(configs, required_interfaces, charm_func=None, services=None, ports=None): """ Set workload status based on complete contexts. status-set missing or incomplete contexts and juju-log details of missing required data. charm_func is a charm specific function to run checking for charm specific requirements such as a VIP setting. + + This function also checks for whether the services defined are ACTUALLY + running and that the ports they advertise are open and being listened to. + + @param services - OPTIONAL: a [{'service': , 'ports': []] + The ports are optional. + If services is a [] then ports are ignored. + @param ports - OPTIONAL: an [] representing ports that shoudl be + open. + @returns None """ incomplete_rel_data = incomplete_relation_data(configs, required_interfaces) state = 'active' @@ -913,6 +957,65 @@ def set_os_workload_status(configs, required_interfaces, charm_func=None): else: message = charm_message + # If the charm thinks the unit is active, check that the actual services + # really are active. + if services is not None and state == 'active': + # if we're passed the dict() then just grab the values as a list. + if isinstance(services, dict): + services = services.values() + # either extract the list of services from the dictionary, or if + # it is a simple string, use that. i.e. works with mixed lists. + _s = [] + for s in services: + if isinstance(s, dict) and 'service' in s: + _s.append(s['service']) + if isinstance(s, str): + _s.append(s) + services_running = [service_running(s) for s in _s] + if not all(services_running): + not_running = [s for s, running in zip(_s, services_running) + if not running] + message = ("Services not running that should be: {}" + .format(", ".join(not_running))) + state = 'blocked' + # also verify that the ports that should be open are open + # NB, that ServiceManager objects only OPTIONALLY have ports + port_map = OrderedDict([(s['service'], s['ports']) + for s in services if 'ports' in s]) + if state == 'active' and port_map: + all_ports = list(itertools.chain(*port_map.values())) + ports_open = [port_has_listener('0.0.0.0', p) + for p in all_ports] + if not all(ports_open): + not_opened = [p for p, opened in zip(all_ports, ports_open) + if not opened] + map_not_open = OrderedDict() + for service, ports in port_map.items(): + closed_ports = set(ports).intersection(not_opened) + if closed_ports: + map_not_open[service] = closed_ports + # find which service has missing ports. They are in service + # order which makes it a bit easier. + message = ( + "Services with ports not open that should be: {}" + .format( + ", ".join([ + "{}: [{}]".format( + service, + ", ".join([str(v) for v in ports])) + for service, ports in map_not_open.items()]))) + state = 'blocked' + + if ports is not None and state == 'active': + # and we can also check ports which we don't know the service for + ports_open = [port_has_listener('0.0.0.0', p) for p in ports] + if not all(ports_open): + message = ( + "Ports which should be open, but are not: {}" + .format(", ".join([str(p) for p, v in zip(ports, ports_open) + if not v]))) + state = 'blocked' + # Set to active if all requirements have been met if state == 'active': message = "Unit is ready" diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8dcd6dd0..a2411c37 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -19,20 +19,35 @@ import os import subprocess +import sys from charmhelpers.fetch import apt_install, apt_update from charmhelpers.core.hookenv import charm_dir, log -try: - from pip import main as pip_execute -except ImportError: - apt_update() - apt_install('python-pip') - from pip import main as pip_execute - __author__ = "Jorge Niedbalski " +def pip_execute(*args, **kwargs): + """Overriden pip_execute() to stop sys.path being changed. + + The act of importing main from the pip module seems to cause add wheels + from the /usr/share/python-wheels which are installed by various tools. + This function ensures that sys.path remains the same after the call is + executed. + """ + try: + _path = sys.path + try: + from pip import main as _pip_execute + except ImportError: + apt_update() + apt_install('python-pip') + from pip import main as _pip_execute + _pip_execute(*args, **kwargs) + finally: + sys.path = _path + + def parse_options(given, available): """Given a set of options, check if available""" for key, value in sorted(given.items()): diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 60ae52b8..fb1bee34 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -120,6 +120,7 @@ class PoolCreationError(Exception): """ A custom error to inform the caller that a pool creation failed. Provides an error message """ + def __init__(self, message): super(PoolCreationError, self).__init__(message) @@ -129,6 +130,7 @@ class Pool(object): An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). """ + def __init__(self, service, name): self.service = service self.name = name @@ -180,36 +182,41 @@ class Pool(object): :return: int. The number of pgs to use. """ validator(value=pool_size, valid_type=int) - osds = get_osds(self.service) - if not osds: + osd_list = get_osds(self.service) + if not osd_list: # NOTE(james-page): Default to 200 for older ceph versions # which don't support OSD query from cli return 200 + osd_list_length = len(osd_list) # Calculate based on Ceph best practices - if osds < 5: + if osd_list_length < 5: return 128 - elif 5 < osds < 10: + elif 5 < osd_list_length < 10: return 512 - elif 10 < osds < 50: + elif 10 < osd_list_length < 50: return 4096 else: - estimate = (osds * 100) / pool_size + estimate = (osd_list_length * 100) / pool_size # Return the next nearest power of 2 index = bisect.bisect_right(powers_of_two, estimate) return powers_of_two[index] class ReplicatedPool(Pool): - def __init__(self, service, name, replicas=2): + def __init__(self, service, name, pg_num=None, replicas=2): super(ReplicatedPool, self).__init__(service=service, name=name) self.replicas = replicas + if pg_num is None: + self.pg_num = self.get_pgs(self.replicas) + else: + self.pg_num = pg_num def create(self): if not pool_exists(self.service, self.name): # Create it - pgs = self.get_pgs(self.replicas) - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs)] + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num)] try: check_call(cmd) except CalledProcessError: @@ -241,7 +248,7 @@ class ErasurePool(Pool): pgs = self.get_pgs(int(erasure_profile['k']) + int(erasure_profile['m'])) # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', self.name, str(pgs), str(pgs), 'erasure', self.erasure_code_profile] try: check_call(cmd) @@ -322,7 +329,8 @@ def set_pool_quota(service, pool_name, max_bytes): :return: None. Can raise CalledProcessError """ # Set a byte quota on a RADOS pool in ceph. - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', max_bytes] + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, + 'max_bytes', str(max_bytes)] try: check_call(cmd) except CalledProcessError: @@ -343,7 +351,25 @@ def remove_pool_quota(service, pool_name): raise -def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', failure_domain='host', +def remove_erasure_profile(service, profile_name): + """ + Create a new erasure code profile if one does not already exist for it. Updates + the profile if it exists. Please see http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + for more details + :param service: six.string_types. The Ceph user name to run the command under + :param profile_name: six.string_types + :return: None. Can raise CalledProcessError + """ + cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', + profile_name] + try: + check_call(cmd) + except CalledProcessError: + raise + + +def create_erasure_profile(service, profile_name, erasure_plugin_name='jerasure', + failure_domain='host', data_chunks=2, coding_chunks=1, locality=None, durability_estimator=None): """ diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index cbaad10d..d2ede320 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -121,7 +121,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Charms which should use the source config option use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', - 'ceph-osd', 'ceph-radosgw'] + 'ceph-osd', 'ceph-radosgw', 'ceph-mon'] # Charms which can not use openstack-origin, ie. many subordinates no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',