From 642bd51cd90f3be17f7bf6a226878c9ccc68aefc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 9 Sep 2014 13:09:30 +0000 Subject: [PATCH 01/61] Add 0mq support --- charm-helpers-sync.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 68 +++- hooks/charmhelpers/contrib/network/ip.py | 20 +- .../contrib/openstack/amulet/deployment.py | 20 +- .../contrib/openstack/amulet/utils.py | 106 ++++-- .../charmhelpers/contrib/openstack/context.py | 49 ++- hooks/charmhelpers/contrib/openstack/ip.py | 10 +- .../contrib/openstack/templates/haproxy.cfg | 6 +- hooks/charmhelpers/contrib/openstack/utils.py | 10 +- .../contrib/storage/linux/utils.py | 3 + hooks/charmhelpers/core/hookenv.py | 44 ++- hooks/charmhelpers/core/host.py | 43 ++- hooks/charmhelpers/core/services/__init__.py | 2 + hooks/charmhelpers/core/services/base.py | 313 ++++++++++++++++++ hooks/charmhelpers/core/services/helpers.py | 125 +++++++ hooks/charmhelpers/core/templating.py | 51 +++ hooks/charmhelpers/fetch/__init__.py | 63 +++- hooks/neutron_ovs_hooks.py | 17 + hooks/neutron_ovs_utils.py | 8 +- hooks/zeromq-configuration-relation-changed | 1 + hooks/zeromq-configuration-relation-joined | 1 + metadata.yaml | 4 + templates/icehouse/neutron.conf | 2 + templates/parts/zeromq | 6 + 24 files changed, 880 insertions(+), 94 deletions(-) create mode 100644 hooks/charmhelpers/core/services/__init__.py create mode 100644 hooks/charmhelpers/core/services/base.py create mode 100644 hooks/charmhelpers/core/services/helpers.py create mode 100644 hooks/charmhelpers/core/templating.py create mode 120000 hooks/zeromq-configuration-relation-changed create mode 120000 hooks/zeromq-configuration-relation-joined create mode 100644 templates/parts/zeromq diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 8af0007c..30a27fb0 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/0mq destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 505de6b2..7151b1d0 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -6,6 +6,11 @@ # Adam Gandelman # +""" +Helpers for clustering and determining "cluster leadership" and other +clustering-related helpers. +""" + import subprocess import os @@ -19,6 +24,7 @@ from charmhelpers.core.hookenv import ( config as config_get, INFO, ERROR, + WARNING, unit_get, ) @@ -27,6 +33,29 @@ class HAIncompleteConfig(Exception): pass +def is_elected_leader(resource): + """ + Returns True if the charm executing this is the elected cluster leader. + + It relies on two mechanisms to determine leadership: + 1. If the charm is part of a corosync cluster, call corosync to + determine leadership. + 2. If the charm is not part of a corosync cluster, the leader is + determined as being "the alive unit with the lowest unit numer". In + other words, the oldest surviving unit. + """ + if is_clustered(): + if not is_crm_leader(resource): + log('Deferring action to CRM leader.', level=INFO) + return False + else: + peers = peer_units() + if peers and not oldest_peer(peers): + log('Deferring action to oldest service unit.', level=INFO) + return False + return True + + def is_clustered(): for r_id in (relation_ids('ha') or []): for unit in (relation_list(r_id) or []): @@ -38,7 +67,11 @@ def is_clustered(): return False -def is_leader(resource): +def is_crm_leader(resource): + """ + Returns True if the charm calling this is the elected corosync leader, + as returned by calling the external "crm" command. + """ cmd = [ "crm", "resource", "show", resource @@ -54,15 +87,31 @@ def is_leader(resource): return False -def peer_units(): +def is_leader(resource): + log("is_leader is deprecated. Please consider using is_crm_leader " + "instead.", level=WARNING) + return is_crm_leader(resource) + + +def peer_units(peer_relation="cluster"): peers = [] - for r_id in (relation_ids('cluster') or []): + for r_id in (relation_ids(peer_relation) or []): for unit in (relation_list(r_id) or []): peers.append(unit) return peers +def peer_ips(peer_relation='cluster', addr_key='private-address'): + '''Return a dict of peers and their private-address''' + peers = {} + for r_id in relation_ids(peer_relation): + for unit in relation_list(r_id): + peers[unit] = relation_get(addr_key, rid=r_id, unit=unit) + return peers + + def oldest_peer(peers): + """Determines who the oldest peer is by comparing unit numbers.""" local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1]) for peer in peers: remote_unit_no = int(peer.split('/')[1]) @@ -72,16 +121,9 @@ def oldest_peer(peers): def eligible_leader(resource): - if is_clustered(): - if not is_leader(resource): - log('Deferring action to CRM leader.', level=INFO) - return False - else: - peers = peer_units() - if peers and not oldest_peer(peers): - log('Deferring action to oldest service unit.', level=INFO) - return False - return True + log("eligible_leader is deprecated. Please consider using " + "is_elected_leader instead.", level=WARNING) + return is_elected_leader(resource) def https(): diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 0972e91a..7edbcc48 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -4,7 +4,7 @@ from functools import partial from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - ERROR, log, + ERROR, log, config, ) try: @@ -154,3 +154,21 @@ def _get_for_address(address, key): get_iface_for_address = partial(_get_for_address, key='iface') get_netmask_for_address = partial(_get_for_address, key='netmask') + + +def get_ipv6_addr(iface="eth0"): + try: + iface_addrs = netifaces.ifaddresses(iface) + if netifaces.AF_INET6 not in iface_addrs: + raise Exception("Interface '%s' doesn't have an ipv6 address." % iface) + + addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6] + ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80') + and config('vip') != a['addr']] + if not ipv6_addr: + raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + + return ipv6_addr[0] + + except ValueError: + raise ValueError("Invalid interface '%s'" % iface) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index e476b6f2..9179eeb1 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -4,8 +4,11 @@ from charmhelpers.contrib.amulet.deployment import ( class OpenStackAmuletDeployment(AmuletDeployment): - """This class inherits from AmuletDeployment and has additional support - that is specifically for use by OpenStack charms.""" + """OpenStack amulet deployment. + + This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms. + """ def __init__(self, series=None, openstack=None, source=None): """Initialize the deployment environment.""" @@ -40,11 +43,14 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.d.configure(service, config) def _get_openstack_release(self): - """Return an integer representing the enum value of the openstack - release.""" - self.precise_essex, self.precise_folsom, self.precise_grizzly, \ - self.precise_havana, self.precise_icehouse, \ - self.trusty_icehouse = range(6) + """Get openstack release. + + Return an integer representing the enum value of the openstack + release. + """ + (self.precise_essex, self.precise_folsom, self.precise_grizzly, + self.precise_havana, self.precise_icehouse, + self.trusty_icehouse) = range(6) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 222281e3..bd327bdc 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -16,8 +16,11 @@ ERROR = logging.ERROR class OpenStackAmuletUtils(AmuletUtils): - """This class inherits from AmuletUtils and has additional support - that is specifically for use by OpenStack charms.""" + """OpenStack amulet utilities. + + This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms. + """ def __init__(self, log_level=ERROR): """Initialize the deployment environment.""" @@ -25,13 +28,17 @@ class OpenStackAmuletUtils(AmuletUtils): def validate_endpoint_data(self, endpoints, admin_port, internal_port, public_port, expected): - """Validate actual endpoint data vs expected endpoint data. The ports - are used to find the matching endpoint.""" + """Validate endpoint data. + + Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint. + """ found = False for ep in endpoints: self.log.debug('endpoint: {}'.format(repr(ep))) - if admin_port in ep.adminurl and internal_port in ep.internalurl \ - and public_port in ep.publicurl: + if (admin_port in ep.adminurl and + internal_port in ep.internalurl and + public_port in ep.publicurl): found = True actual = {'id': ep.id, 'region': ep.region, @@ -47,8 +54,11 @@ class OpenStackAmuletUtils(AmuletUtils): return 'endpoint not found' def validate_svc_catalog_endpoint_data(self, expected, actual): - """Validate a list of actual service catalog endpoints vs a list of - expected service catalog endpoints.""" + """Validate service catalog endpoint data. + + Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints. + """ self.log.debug('actual: {}'.format(repr(actual))) for k, v in expected.iteritems(): if k in actual: @@ -60,8 +70,11 @@ class OpenStackAmuletUtils(AmuletUtils): return ret def validate_tenant_data(self, expected, actual): - """Validate a list of actual tenant data vs list of expected tenant - data.""" + """Validate tenant data. + + Validate a list of actual tenant data vs list of expected tenant + data. + """ self.log.debug('actual: {}'.format(repr(actual))) for e in expected: found = False @@ -78,8 +91,11 @@ class OpenStackAmuletUtils(AmuletUtils): return ret def validate_role_data(self, expected, actual): - """Validate a list of actual role data vs a list of expected role - data.""" + """Validate role data. + + Validate a list of actual role data vs a list of expected role + data. + """ self.log.debug('actual: {}'.format(repr(actual))) for e in expected: found = False @@ -95,8 +111,11 @@ class OpenStackAmuletUtils(AmuletUtils): return ret def validate_user_data(self, expected, actual): - """Validate a list of actual user data vs a list of expected user - data.""" + """Validate user data. + + Validate a list of actual user data vs a list of expected user + data. + """ self.log.debug('actual: {}'.format(repr(actual))) for e in expected: found = False @@ -114,21 +133,24 @@ class OpenStackAmuletUtils(AmuletUtils): return ret def validate_flavor_data(self, expected, actual): - """Validate a list of actual flavors vs a list of expected flavors.""" + """Validate flavor data. + + Validate a list of actual flavors vs a list of expected flavors. + """ self.log.debug('actual: {}'.format(repr(actual))) act = [a.name for a in actual] return self._validate_list_data(expected, act) def tenant_exists(self, keystone, tenant): - """Return True if tenant exists""" + """Return True if tenant exists.""" return tenant in [t.name for t in keystone.tenants.list()] def authenticate_keystone_admin(self, keystone_sentry, user, password, tenant): """Authenticates admin user with the keystone admin endpoint.""" - service_ip = \ - keystone_sentry.relation('shared-db', - 'mysql:shared-db')['private-address'] + unit = keystone_sentry + service_ip = unit.relation('shared-db', + 'mysql:shared-db')['private-address'] ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) return keystone_client.Client(username=user, password=password, tenant_name=tenant, auth_url=ep) @@ -177,12 +199,40 @@ class OpenStackAmuletUtils(AmuletUtils): image = glance.images.create(name=image_name, is_public=True, disk_format='qcow2', container_format='bare', data=f) + count = 1 + status = image.status + while status != 'active' and count < 10: + time.sleep(3) + image = glance.images.get(image.id) + status = image.status + self.log.debug('image status: {}'.format(status)) + count += 1 + + if status != 'active': + self.log.error('image creation timed out') + return None + return image def delete_image(self, glance, image): """Delete the specified image.""" + num_before = len(list(glance.images.list())) glance.images.delete(image) + count = 1 + num_after = len(list(glance.images.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(glance.images.list())) + self.log.debug('number of images: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('image deletion timed out') + return False + + return True + def create_instance(self, nova, image_name, instance_name, flavor): """Create the specified instance.""" image = nova.images.find(name=image_name) @@ -199,11 +249,27 @@ class OpenStackAmuletUtils(AmuletUtils): self.log.debug('instance status: {}'.format(status)) count += 1 - if status == 'BUILD': + if status != 'ACTIVE': + self.log.error('instance creation timed out') return None return instance def delete_instance(self, nova, instance): """Delete the specified instance.""" + num_before = len(list(nova.servers.list())) nova.servers.delete(instance) + + count = 1 + num_after = len(list(nova.servers.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(nova.servers.list())) + self.log.debug('number of instances: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('instance deletion timed out') + return False + + return True diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 92c41b23..988bef19 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, + is_relation_made, relation_set, unit_get, unit_private_ip, @@ -44,7 +45,10 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) -from charmhelpers.contrib.network.ip import get_address_in_network +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + get_ipv6_addr, +) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -401,9 +405,12 @@ class HAProxyContext(OSContextGenerator): cluster_hosts = {} l_unit = local_unit().replace('/', '-') - cluster_hosts[l_unit] = \ - get_address_in_network(config('os-internal-network'), - unit_get('private-address')) + if config('prefer-ipv6'): + addr = get_ipv6_addr() + else: + addr = unit_get('private-address') + cluster_hosts[l_unit] = get_address_in_network(config('os-internal-network'), + addr) for rid in relation_ids('cluster'): for unit in related_units(rid): @@ -414,6 +421,16 @@ class HAProxyContext(OSContextGenerator): ctxt = { 'units': cluster_hosts, } + + if config('prefer-ipv6'): + ctxt['local_host'] = 'ip6-localhost' + ctxt['haproxy_host'] = '::' + ctxt['stat_port'] = ':::8888' + else: + ctxt['local_host'] = '127.0.0.1' + ctxt['haproxy_host'] = '0.0.0.0' + ctxt['stat_port'] = ':8888' + if len(cluster_hosts.keys()) > 1: # Enable haproxy when we have enough peers. log('Ensuring haproxy enabled in /etc/default/haproxy.') @@ -753,6 +770,17 @@ class SubordinateConfigContext(OSContextGenerator): return ctxt +class LogLevelContext(OSContextGenerator): + + def __call__(self): + ctxt = {} + ctxt['debug'] = \ + False if config('debug') is None else config('debug') + ctxt['verbose'] = \ + False if config('verbose') is None else config('verbose') + return ctxt + + class SyslogContext(OSContextGenerator): def __call__(self): @@ -760,3 +788,16 @@ class SyslogContext(OSContextGenerator): 'use_syslog': config('use-syslog') } return ctxt + + +class ZeroMQContext(OSContextGenerator): + interfaces = ['zeromq-configuration'] + + def __call__(self): + ctxt = {} + if is_relation_made('zeromq-configuration', 'host'): + for rid in relation_ids('zeromq-configuration'): + for unit in related_units(rid): + ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) + ctxt['zmq_host'] = relation_get('host', unit, rid) + return ctxt \ No newline at end of file diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 7e7a536f..affe8cd1 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -7,6 +7,7 @@ from charmhelpers.contrib.network.ip import ( get_address_in_network, is_address_in_network, is_ipv6, + get_ipv6_addr, ) from charmhelpers.contrib.hahelpers.cluster import is_clustered @@ -64,10 +65,13 @@ def resolve_address(endpoint_type=PUBLIC): vip): resolved_address = vip else: + if config('prefer-ipv6'): + fallback_addr = get_ipv6_addr() + else: + fallback_addr = unit_get(_address_map[endpoint_type]['fallback']) resolved_address = get_address_in_network( - config(_address_map[endpoint_type]['config']), - unit_get(_address_map[endpoint_type]['fallback']) - ) + config(_address_map[endpoint_type]['config']), fallback_addr) + if resolved_address is None: raise ValueError('Unable to resolve a suitable IP address' ' based on charm state and configuration') diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index a95eddd1..ce0e2738 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -1,6 +1,6 @@ global - log 127.0.0.1 local0 - log 127.0.0.1 local1 notice + log {{ local_host }} local0 + log {{ local_host }} local1 notice maxconn 20000 user haproxy group haproxy @@ -17,7 +17,7 @@ defaults timeout client 30000 timeout server 30000 -listen stats :8888 +listen stats {{ stat_port }} mode http stats enable stats hide-version diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 127b03fe..23d237de 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,7 +23,7 @@ from charmhelpers.contrib.storage.linux.lvm import ( ) from charmhelpers.core.host import lsb_release, mounts, umount -from charmhelpers.fetch import apt_install +from charmhelpers.fetch import apt_install, apt_cache from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -70,6 +70,7 @@ SWIFT_CODENAMES = OrderedDict([ ('1.13.0', 'icehouse'), ('1.12.0', 'icehouse'), ('1.11.0', 'icehouse'), + ('2.0.0', 'juno'), ]) DEFAULT_LOOPBACK_SIZE = '5G' @@ -134,13 +135,8 @@ def get_os_version_codename(codename): def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' import apt_pkg as apt - apt.init() - # Tell apt to build an in-memory cache to prevent race conditions (if - # another process is already building the cache). - apt.config.set("Dir::Cache::pkgcache", "") - - cache = apt.Cache() + cache = apt_cache() try: pkg = cache[package] diff --git a/hooks/charmhelpers/contrib/storage/linux/utils.py b/hooks/charmhelpers/contrib/storage/linux/utils.py index 8d0f6116..1b958712 100644 --- a/hooks/charmhelpers/contrib/storage/linux/utils.py +++ b/hooks/charmhelpers/contrib/storage/linux/utils.py @@ -46,5 +46,8 @@ def is_device_mounted(device): :returns: boolean: True if the path represents a mounted device, False if it doesn't. ''' + is_partition = bool(re.search(r".*[0-9]+\b", device)) out = check_output(['mount']) + if is_partition: + return bool(re.search(device + r"\b", out)) return bool(re.search(device + r"[0-9]+\b", out)) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c9530433..f396e03a 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -156,12 +156,15 @@ def hook_name(): class Config(dict): - """A Juju charm config dictionary that can write itself to - disk (as json) and track which values have changed since - the previous hook invocation. + """A dictionary representation of the charm's config.yaml, with some + extra features: - Do not instantiate this object directly - instead call - ``hookenv.config()`` + - See which values in the dictionary have changed since the previous hook. + - For values that have changed, see what the previous value was. + - Store arbitrary data for use in a later hook. + + NOTE: Do not instantiate this object directly - instead call + ``hookenv.config()``, which will return an instance of :class:`Config`. Example usage:: @@ -170,8 +173,8 @@ class Config(dict): >>> config = hookenv.config() >>> config['foo'] 'bar' + >>> # store a new key/value for later use >>> config['mykey'] = 'myval' - >>> config.save() >>> # user runs `juju set mycharm foo=baz` @@ -188,22 +191,23 @@ class Config(dict): >>> # keys/values that we add are preserved across hooks >>> config['mykey'] 'myval' - >>> # don't forget to save at the end of hook! - >>> config.save() """ CONFIG_FILE_NAME = '.juju-persistent-config' def __init__(self, *args, **kw): super(Config, self).__init__(*args, **kw) + self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) if os.path.exists(self.path): self.load_previous() def load_previous(self, path=None): - """Load previous copy of config from disk so that current values - can be compared to previous values. + """Load previous copy of config from disk. + + In normal usage you don't need to call this method directly - it + is called automatically at object initialization. :param path: @@ -218,8 +222,8 @@ class Config(dict): self._prev_dict = json.load(f) def changed(self, key): - """Return true if the value for this key has changed since - the last save. + """Return True if the current value for this key is different from + the previous value. """ if self._prev_dict is None: @@ -228,7 +232,7 @@ class Config(dict): def previous(self, key): """Return previous value for this key, or None if there - is no "previous" value. + is no previous value. """ if self._prev_dict: @@ -238,7 +242,13 @@ class Config(dict): def save(self): """Save this config to disk. - Preserves items in _prev_dict that do not exist in self. + If the charm is using the :mod:`Services Framework ` + or :meth:'@hook ' decorator, this + is called automatically at the end of successful hook execution. + Otherwise, it should be called directly by user code. + + To disable automatic saves, set ``implicit_save=False`` on this + instance. """ if self._prev_dict: @@ -285,8 +295,9 @@ def relation_get(attribute=None, unit=None, rid=None): raise -def relation_set(relation_id=None, relation_settings={}, **kwargs): +def relation_set(relation_id=None, relation_settings=None, **kwargs): """Set relation information for the current unit""" + relation_settings = relation_settings if relation_settings else {} relation_cmd_line = ['relation-set'] if relation_id is not None: relation_cmd_line.extend(('-r', relation_id)) @@ -477,6 +488,9 @@ class Hooks(object): hook_name = os.path.basename(args[0]) if hook_name in self._hooks: self._hooks[hook_name]() + cfg = config() + if cfg.implicit_save: + cfg.save() else: raise UnregisteredHookError(hook_name) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index d934f940..b85b0280 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -12,6 +12,8 @@ import random import string import subprocess import hashlib +import shutil +from contextlib import contextmanager from collections import OrderedDict @@ -52,7 +54,7 @@ def service(action, service_name): def service_running(service): """Determine whether a system service is running""" try: - output = subprocess.check_output(['service', service, 'status']) + output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError: return False else: @@ -62,6 +64,16 @@ def service_running(service): return False +def service_available(service_name): + """Determine whether a system service is available""" + try: + subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return False + else: + return True + + def adduser(username, password=None, shell='/bin/bash', system_user=False): """Add a user to the system""" try: @@ -320,12 +332,29 @@ def cmp_pkgrevno(package, revno, pkgcache=None): ''' import apt_pkg + from charmhelpers.fetch import apt_cache if not pkgcache: - apt_pkg.init() - # Force Apt to build its cache in memory. That way we avoid race - # conditions with other applications building the cache in the same - # place. - apt_pkg.config.set("Dir::Cache::pkgcache", "") - pkgcache = apt_pkg.Cache() + pkgcache = apt_cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) + + +@contextmanager +def chdir(d): + cur = os.getcwd() + try: + yield os.chdir(d) + finally: + os.chdir(cur) + + +def chownr(path, owner, group): + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + + for root, dirs, files in os.walk(path): + for name in dirs + files: + full = os.path.join(root, name) + broken_symlink = os.path.lexists(full) and not os.path.exists(full) + if not broken_symlink: + os.chown(full, uid, gid) diff --git a/hooks/charmhelpers/core/services/__init__.py b/hooks/charmhelpers/core/services/__init__.py new file mode 100644 index 00000000..e8039a84 --- /dev/null +++ b/hooks/charmhelpers/core/services/__init__.py @@ -0,0 +1,2 @@ +from .base import * +from .helpers import * diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py new file mode 100644 index 00000000..87ecb130 --- /dev/null +++ b/hooks/charmhelpers/core/services/base.py @@ -0,0 +1,313 @@ +import os +import re +import json +from collections import Iterable + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +__all__ = ['ServiceManager', 'ManagerCallback', + 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports', + 'service_restart', 'service_stop'] + + +class ServiceManager(object): + def __init__(self, services=None): + """ + Register a list of services, given their definitions. + + Service definitions are dicts in the following formats (all keys except + 'service' are optional):: + + { + "service": , + "required_data": , + "provided_data": , + "data_ready": , + "data_lost": , + "start": , + "stop": , + "ports": , + } + + The 'required_data' list should contain dicts of required data (or + dependency managers that act like dicts and know how to collect the data). + Only when all items in the 'required_data' list are populated are the list + of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more + information. + + The 'provided_data' list should contain relation data providers, most likely + a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, + that will indicate a set of data to set on a given relation. + + The 'data_ready' value should be either a single callback, or a list of + callbacks, to be called when all items in 'required_data' pass `is_ready()`. + Each callback will be called with the service name as the only parameter. + After all of the 'data_ready' callbacks are called, the 'start' callbacks + are fired. + + The 'data_lost' value should be either a single callback, or a list of + callbacks, to be called when a 'required_data' item no longer passes + `is_ready()`. Each callback will be called with the service name as the + only parameter. After all of the 'data_lost' callbacks are called, + the 'stop' callbacks are fired. + + The 'start' value should be either a single callback, or a list of + callbacks, to be called when starting the service, after the 'data_ready' + callbacks are complete. Each callback will be called with the service + name as the only parameter. This defaults to + `[host.service_start, services.open_ports]`. + + The 'stop' value should be either a single callback, or a list of + callbacks, to be called when stopping the service. If the service is + being stopped because it no longer has all of its 'required_data', this + will be called after all of the 'data_lost' callbacks are complete. + Each callback will be called with the service name as the only parameter. + This defaults to `[services.close_ports, host.service_stop]`. + + The 'ports' value should be a list of ports to manage. The default + 'start' handler will open the ports after the service is started, + and the default 'stop' handler will close the ports prior to stopping + the service. + + + Examples: + + The following registers an Upstart service called bingod that depends on + a mongodb relation and which runs a custom `db_migrate` function prior to + restarting the service, and a Runit service called spadesd:: + + manager = services.ServiceManager([ + { + 'service': 'bingod', + 'ports': [80, 443], + 'required_data': [MongoRelation(), config(), {'my': 'data'}], + 'data_ready': [ + services.template(source='bingod.conf'), + services.template(source='bingod.ini', + target='/etc/bingod.ini', + owner='bingo', perms=0400), + ], + }, + { + 'service': 'spadesd', + 'data_ready': services.template(source='spadesd_run.j2', + target='/etc/sv/spadesd/run', + perms=0555), + 'start': runit_start, + 'stop': runit_stop, + }, + ]) + manager.manage() + """ + self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') + self._ready = None + self.services = {} + for service in services or []: + service_name = service['service'] + self.services[service_name] = service + + def manage(self): + """ + Handle the current hook by doing The Right Thing with the registered services. + """ + hook_name = hookenv.hook_name() + if hook_name == 'stop': + self.stop_services() + else: + self.provide_data() + self.reconfigure_services() + cfg = hookenv.config() + if cfg.implicit_save: + cfg.save() + + def provide_data(self): + """ + Set the relation data for each provider in the ``provided_data`` list. + + A provider must have a `name` attribute, which indicates which relation + to set data on, and a `provide_data()` method, which returns a dict of + data to set. + """ + hook_name = hookenv.hook_name() + for service in self.services.values(): + for provider in service.get('provided_data', []): + if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name): + data = provider.provide_data() + _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data + if _ready: + hookenv.relation_set(None, data) + + def reconfigure_services(self, *service_names): + """ + Update all files for one or more registered services, and, + if ready, optionally restart them. + + If no service names are given, reconfigures all registered services. + """ + for service_name in service_names or self.services.keys(): + if self.is_ready(service_name): + self.fire_event('data_ready', service_name) + self.fire_event('start', service_name, default=[ + service_restart, + manage_ports]) + self.save_ready(service_name) + else: + if self.was_ready(service_name): + self.fire_event('data_lost', service_name) + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + self.save_lost(service_name) + + def stop_services(self, *service_names): + """ + Stop one or more registered services, by name. + + If no service names are given, stops all registered services. + """ + for service_name in service_names or self.services.keys(): + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + + def get_service(self, service_name): + """ + Given the name of a registered service, return its service definition. + """ + service = self.services.get(service_name) + if not service: + raise KeyError('Service not registered: %s' % service_name) + return service + + def fire_event(self, event_name, service_name, default=None): + """ + Fire a data_ready, data_lost, start, or stop event on a given service. + """ + service = self.get_service(service_name) + callbacks = service.get(event_name, default) + if not callbacks: + return + if not isinstance(callbacks, Iterable): + callbacks = [callbacks] + for callback in callbacks: + if isinstance(callback, ManagerCallback): + callback(self, service_name, event_name) + else: + callback(service_name) + + def is_ready(self, service_name): + """ + Determine if a registered service is ready, by checking its 'required_data'. + + A 'required_data' item can be any mapping type, and is considered ready + if `bool(item)` evaluates as True. + """ + service = self.get_service(service_name) + reqs = service.get('required_data', []) + return all(bool(req) for req in reqs) + + def _load_ready_file(self): + if self._ready is not None: + return + if os.path.exists(self._ready_file): + with open(self._ready_file) as fp: + self._ready = set(json.load(fp)) + else: + self._ready = set() + + def _save_ready_file(self): + if self._ready is None: + return + with open(self._ready_file, 'w') as fp: + json.dump(list(self._ready), fp) + + def save_ready(self, service_name): + """ + Save an indicator that the given service is now data_ready. + """ + self._load_ready_file() + self._ready.add(service_name) + self._save_ready_file() + + def save_lost(self, service_name): + """ + Save an indicator that the given service is no longer data_ready. + """ + self._load_ready_file() + self._ready.discard(service_name) + self._save_ready_file() + + def was_ready(self, service_name): + """ + Determine if the given service was previously data_ready. + """ + self._load_ready_file() + return service_name in self._ready + + +class ManagerCallback(object): + """ + Special case of a callback that takes the `ServiceManager` instance + in addition to the service name. + + Subclasses should implement `__call__` which should accept three parameters: + + * `manager` The `ServiceManager` instance + * `service_name` The name of the service it's being triggered for + * `event_name` The name of the event that this callback is handling + """ + def __call__(self, manager, service_name, event_name): + raise NotImplementedError() + + +class PortManagerCallback(ManagerCallback): + """ + Callback class that will open or close ports, for use as either + a start or stop action. + """ + def __call__(self, manager, service_name, event_name): + service = manager.get_service(service_name) + new_ports = service.get('ports', []) + port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name)) + if os.path.exists(port_file): + with open(port_file) as fp: + old_ports = fp.read().split(',') + for old_port in old_ports: + if bool(old_port): + old_port = int(old_port) + if old_port not in new_ports: + hookenv.close_port(old_port) + with open(port_file, 'w') as fp: + fp.write(','.join(str(port) for port in new_ports)) + for port in new_ports: + if event_name == 'start': + hookenv.open_port(port) + elif event_name == 'stop': + hookenv.close_port(port) + + +def service_stop(service_name): + """ + Wrapper around host.service_stop to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_running(service_name): + host.service_stop(service_name) + + +def service_restart(service_name): + """ + Wrapper around host.service_restart to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_available(service_name): + if host.service_running(service_name): + host.service_restart(service_name) + else: + host.service_start(service_name) + + +# Convenience aliases +open_ports = close_ports = manage_ports = PortManagerCallback() diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py new file mode 100644 index 00000000..4b90589b --- /dev/null +++ b/hooks/charmhelpers/core/services/helpers.py @@ -0,0 +1,125 @@ +from charmhelpers.core import hookenv +from charmhelpers.core import templating + +from charmhelpers.core.services.base import ManagerCallback + + +__all__ = ['RelationContext', 'TemplateCallback', + 'render_template', 'template'] + + +class RelationContext(dict): + """ + Base class for a context generator that gets relation data from juju. + + Subclasses must provide the attributes `name`, which is the name of the + interface of interest, `interface`, which is the type of the interface of + interest, and `required_keys`, which is the set of keys required for the + relation to be considered complete. The data for all interfaces matching + the `name` attribute that are complete will used to populate the dictionary + values (see `get_data`, below). + + The generated context will be namespaced under the interface type, to prevent + potential naming conflicts. + """ + name = None + interface = None + required_keys = [] + + def __init__(self, *args, **kwargs): + super(RelationContext, self).__init__(*args, **kwargs) + self.get_data() + + def __bool__(self): + """ + Returns True if all of the required_keys are available. + """ + return self.is_ready() + + __nonzero__ = __bool__ + + def __repr__(self): + return super(RelationContext, self).__repr__() + + def is_ready(self): + """ + Returns True if all of the `required_keys` are available from any units. + """ + ready = len(self.get(self.name, [])) > 0 + if not ready: + hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG) + return ready + + def _is_ready(self, unit_data): + """ + Helper method that tests a set of relation data and returns True if + all of the `required_keys` are present. + """ + return set(unit_data.keys()).issuperset(set(self.required_keys)) + + def get_data(self): + """ + Retrieve the relation data for each unit involved in a relation and, + if complete, store it in a list under `self[self.name]`. This + is automatically called when the RelationContext is instantiated. + + The units are sorted lexographically first by the service ID, then by + the unit ID. Thus, if an interface has two other services, 'db:1' + and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1', + and 'db:2' having one unit, 'mediawiki/0', all of which have a complete + set of data, the relation data for the units will be stored in the + order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'. + + If you only care about a single unit on the relation, you can just + access it as `{{ interface[0]['key'] }}`. However, if you can at all + support multiple units on a relation, you should iterate over the list, + like:: + + {% for unit in interface -%} + {{ unit['key'] }}{% if not loop.last %},{% endif %} + {%- endfor %} + + Note that since all sets of relation data from all related services and + units are in a single list, if you need to know which service or unit a + set of data came from, you'll need to extend this class to preserve + that information. + """ + if not hookenv.relation_ids(self.name): + return + + ns = self.setdefault(self.name, []) + for rid in sorted(hookenv.relation_ids(self.name)): + for unit in sorted(hookenv.related_units(rid)): + reldata = hookenv.relation_get(rid=rid, unit=unit) + if self._is_ready(reldata): + ns.append(reldata) + + def provide_data(self): + """ + Return data to be relation_set for this interface. + """ + return {} + + +class TemplateCallback(ManagerCallback): + """ + Callback class that will render a template, for use as a ready action. + """ + def __init__(self, source, target, owner='root', group='root', perms=0444): + self.source = source + self.target = target + self.owner = owner + self.group = group + self.perms = perms + + def __call__(self, manager, service_name, event_name): + service = manager.get_service(service_name) + context = {} + for ctx in service.get('required_data', []): + context.update(ctx) + templating.render(self.source, self.target, context, + self.owner, self.group, self.perms) + + +# Convenience aliases for templates +render_template = template = TemplateCallback diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py new file mode 100644 index 00000000..2c638853 --- /dev/null +++ b/hooks/charmhelpers/core/templating.py @@ -0,0 +1,51 @@ +import os + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None): + """ + Render a template. + + The `source` path, if not absolute, is relative to the `templates_dir`. + + The `target` path should be absolute. + + The context should be a dict containing the values to be replaced in the + template. + + The `owner`, `group`, and `perms` options will be passed to `write_file`. + + If omitted, `templates_dir` defaults to the `templates` folder in the charm. + + Note: Using this requires python-jinja2; if it is not installed, calling + this will attempt to use charmhelpers.fetch.apt_install to install it. + """ + try: + from jinja2 import FileSystemLoader, Environment, exceptions + except ImportError: + try: + from charmhelpers.fetch import apt_install + except ImportError: + hookenv.log('Could not import jinja2, and could not import ' + 'charmhelpers.fetch to install it', + level=hookenv.ERROR) + raise + apt_install('python-jinja2', fatal=True) + from jinja2 import FileSystemLoader, Environment, exceptions + + if templates_dir is None: + templates_dir = os.path.join(hookenv.charm_dir(), 'templates') + loader = Environment(loader=FileSystemLoader(templates_dir)) + try: + source = source + template = loader.get_template(source) + except exceptions.TemplateNotFound as e: + hookenv.log('Could not load template %s from %s.' % + (source, templates_dir), + level=hookenv.ERROR) + raise e + content = template.render(context) + host.mkdir(os.path.dirname(target)) + host.write_file(target, content, owner, group, perms) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 5be512ce..8e9d3804 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -1,4 +1,5 @@ import importlib +from tempfile import NamedTemporaryFile import time from yaml import safe_load from charmhelpers.core.host import ( @@ -116,14 +117,7 @@ class BaseFetchHandler(object): def filter_installed_packages(packages): """Returns a list of packages that require installation""" - import apt_pkg - apt_pkg.init() - - # Tell apt to build an in-memory cache to prevent race conditions (if - # another process is already building the cache). - apt_pkg.config.set("Dir::Cache::pkgcache", "") - - cache = apt_pkg.Cache() + cache = apt_cache() _pkgs = [] for package in packages: try: @@ -136,6 +130,16 @@ def filter_installed_packages(packages): return _pkgs +def apt_cache(in_memory=True): + """Build and return an apt cache""" + import apt_pkg + apt_pkg.init() + if in_memory: + apt_pkg.config.set("Dir::Cache::pkgcache", "") + apt_pkg.config.set("Dir::Cache::srcpkgcache", "") + return apt_pkg.Cache() + + def apt_install(packages, options=None, fatal=False): """Install one or more packages""" if options is None: @@ -201,6 +205,27 @@ def apt_hold(packages, fatal=False): def add_source(source, key=None): + """Add a package source to this system. + + @param source: a URL or sources.list entry, as supported by + add-apt-repository(1). Examples: + ppa:charmers/example + deb https://stub:key@private.example.com/ubuntu trusty main + + In addition: + 'proposed:' may be used to enable the standard 'proposed' + pocket for the release. + 'cloud:' may be used to activate official cloud archive pockets, + such as 'cloud:icehouse' + + @param key: A key to be added to the system's APT keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. ppa and cloud archive keys + are securely added automtically, so sould not be provided. + """ if source is None: log('Source is not present. Skipping') return @@ -225,10 +250,23 @@ def add_source(source, key=None): release = lsb_release()['DISTRIB_CODENAME'] with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: apt.write(PROPOSED_POCKET.format(release)) + else: + raise SourceConfigError("Unknown source: {!r}".format(source)) + if key: - subprocess.check_call(['apt-key', 'adv', '--keyserver', - 'hkp://keyserver.ubuntu.com:80', '--recv', - key]) + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: + with NamedTemporaryFile() as key_file: + key_file.write(key) + key_file.flush() + key_file.seek(0) + subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) + else: + # Note that hkp: is in no way a secure protocol. Using a + # GPG key id is pointless from a security POV unless you + # absolutely trust your network and DNS. + subprocess.check_call(['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv', + key]) def configure_sources(update=False, @@ -238,7 +276,8 @@ def configure_sources(update=False, Configure multiple sources from charm configuration. The lists are encoded as yaml fragments in the configuration. - The frament needs to be included as a string. + The frament needs to be included as a string. Sources and their + corresponding keys are of the types supported by add_source(). Example config: install_sources: | diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index eb53094d..bc9630ff 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -8,6 +8,7 @@ from charmhelpers.core.hookenv import ( config, log, relation_set, + relation_ids, ) from charmhelpers.core.host import ( @@ -20,6 +21,7 @@ from charmhelpers.fetch import ( from neutron_ovs_utils import ( determine_packages, + get_topics, register_configs, restart_map, ) @@ -42,6 +44,8 @@ def install(): @restart_on_change(restart_map()) def config_changed(): CONFIGS.write_all() + for rid in relation_ids('zeromq-configuration'): + zeromq_configuration_relation_joined(rid) @hooks.hook('amqp-relation-joined') @@ -61,6 +65,19 @@ def amqp_changed(): CONFIGS.write_all() +@hooks.hook('zeromq-configuration-relation-joined') +def zeromq_configuration_relation_joined(relid=None): + relation_set(relation_id=relid, + topics=" ".join(get_topics()), + users="nova") + + +@hooks.hook('zeromq-configuration-relation-changed') +@restart_on_change(restart_map(), stopstart=True) +def zeromq_configuration_relation_changed(): + CONFIGS.write_all() + + def main(): try: hooks.execute(sys.argv) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 367020f9..281880e8 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -18,7 +18,8 @@ BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { 'services': ['neutron-plugin-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext(), - context.AMQPContext()], + context.AMQPContext(), + context.ZeroMQContext()], }), (ML2_CONF, { 'services': ['neutron-plugin-openvswitch-agent'], @@ -56,3 +57,8 @@ def restart_map(): state. ''' return {k: v['services'] for k, v in resource_map().iteritems()} + + +def get_topics(): + return ['q-plugin'] + diff --git a/hooks/zeromq-configuration-relation-changed b/hooks/zeromq-configuration-relation-changed new file mode 120000 index 00000000..55aa8e52 --- /dev/null +++ b/hooks/zeromq-configuration-relation-changed @@ -0,0 +1 @@ +neutron_ovs_hooks.py \ No newline at end of file diff --git a/hooks/zeromq-configuration-relation-joined b/hooks/zeromq-configuration-relation-joined new file mode 120000 index 00000000..55aa8e52 --- /dev/null +++ b/hooks/zeromq-configuration-relation-joined @@ -0,0 +1 @@ +neutron_ovs_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 0e840258..dfbefd7b 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -28,3 +28,7 @@ requires: scope: container neutron-plugin-api: interface: neutron-plugin-api + zeromq-configuration: + interface: zeromq-configuration + scope: container + diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 964f6dce..8e8bf133 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -25,6 +25,8 @@ notification_topics = notifications {% include "parts/rabbitmq" %} +{% include "parts/zeromq" %} + [QUOTAS] [DEFAULT_SERVICETYPE] diff --git a/templates/parts/zeromq b/templates/parts/zeromq new file mode 100644 index 00000000..3e32288c --- /dev/null +++ b/templates/parts/zeromq @@ -0,0 +1,6 @@ +{% if zmq_host -%} +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) +rpc_backend = zmq +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +rpc_zmq_host = {{ zmq_host }} +{% endif -%} From 8424f0cc04ab88ca7665fa0018fa0ef563f5a3f1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 9 Sep 2014 15:22:36 +0000 Subject: [PATCH 02/61] Fix funky neutron semi-oslo settings --- templates/parts/zeromq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/parts/zeromq b/templates/parts/zeromq index 3e32288c..024fa88f 100644 --- a/templates/parts/zeromq +++ b/templates/parts/zeromq @@ -1,6 +1,6 @@ {% if zmq_host -%} # ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) -rpc_backend = zmq -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +rpc_backend = neutron.openstack.common.rpc.impl_zmq +rpc_zmq_matchmaker = neutron.openstack.common.rpc.matchmaker_ring.MatchMakerRing rpc_zmq_host = {{ zmq_host }} {% endif -%} From cfdcc32c3473fd53b6090abe25865022c46599e1 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 9 Sep 2014 17:04:26 +0000 Subject: [PATCH 03/61] Fixup topics --- hooks/neutron_ovs_utils.py | 8 +++++++- templates/parts/zeromq | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 281880e8..3a4fb5cd 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -60,5 +60,11 @@ def restart_map(): def get_topics(): - return ['q-plugin'] + topics = [] + topics.append('q-agent-notifier-port-update') + topics.append('q-agent-notifier-network-delete') + topics.append('q-agent-notifier-tunnel-update') + topics.append('q-agent-notifier-security_group-update') + topics.append('q-agent-notifier-dvr-update') + return topics diff --git a/templates/parts/zeromq b/templates/parts/zeromq index 024fa88f..3e32288c 100644 --- a/templates/parts/zeromq +++ b/templates/parts/zeromq @@ -1,6 +1,6 @@ {% if zmq_host -%} # ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) -rpc_backend = neutron.openstack.common.rpc.impl_zmq -rpc_zmq_matchmaker = neutron.openstack.common.rpc.matchmaker_ring.MatchMakerRing +rpc_backend = zmq +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing rpc_zmq_host = {{ zmq_host }} {% endif -%} From 3867666491d977d17738572028a4fdb346bfe814 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 10 Sep 2014 10:12:06 +0000 Subject: [PATCH 04/61] Update correct user --- hooks/neutron_ovs_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index bc9630ff..75841c9b 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -69,7 +69,7 @@ def amqp_changed(): def zeromq_configuration_relation_joined(relid=None): relation_set(relation_id=relid, topics=" ".join(get_topics()), - users="nova") + users="neutron") @hooks.hook('zeromq-configuration-relation-changed') From 9d1c26e2844ef2de7319ac96760d86be55d872f6 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 10 Sep 2014 17:16:46 +0000 Subject: [PATCH 05/61] Add support for notifications with zmq --- .../charmhelpers/contrib/openstack/context.py | 26 ++++++++++++++++++- hooks/charmhelpers/contrib/openstack/utils.py | 12 ++++++++- hooks/neutron_ovs_utils.py | 3 ++- templates/icehouse/neutron.conf | 2 ++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 988bef19..7c55084d 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -50,6 +50,9 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr, ) +from charmhelpers.contrib.openstack.utils import ( + get_matchmaker_map, +) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -800,4 +803,25 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) - return ctxt \ No newline at end of file + return ctxt + +class NotificationDriverContext(OSContextGenerator): + + def __init__(self, zmq_relation='zeromq-configuration', amqp_relation='amqp'): + """ + :param zmq_relation : Name of Zeromq relation to check + """ + self.zmq_relation = zmq_relation + self.amqp_relation = amqp_relation + + def __call__(self): + ctxt = { + 'notifications': "False", + } + if is_relation_made(self.zmq_relation): + matchmaker_data = get_matchmaker_map() + if 'notifications-info' in matchmaker_data: + ctxt['notifications'] = "True" + elif is_relation_made(self.amqp_relation): + ctxt['notifications'] = "True" + return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 23d237de..4bcd3cff 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -3,15 +3,17 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict -import subprocess +import json import os import socket +import subprocess import sys from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, + is_relation_made, ERROR, INFO ) @@ -457,3 +459,11 @@ def get_hostname(address, fqdn=True): return result else: return result.split('.')[0] + + +def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): + mm_map = {} + if os.path.isfile(mm_file): + with open(mm_file, 'r') as f: + mm_map = json.load(f) + return mm_map diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 3a4fb5cd..e1cb16fd 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -19,7 +19,8 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-plugin-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext(), context.AMQPContext(), - context.ZeroMQContext()], + context.ZeroMQContext(), + context.NotificationDriverContext()], }), (ML2_CONF, { 'services': ['neutron-plugin-openvswitch-agent'], diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 8e8bf133..c82ab09d 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -19,7 +19,9 @@ core_plugin = {{ core_plugin }} api_paste_config = /etc/neutron/api-paste.ini auth_strategy = keystone +{% if notifications == 'True' -%} notification_driver = neutron.openstack.common.notifier.rpc_notifier +{% endif -%} default_notification_level = INFO notification_topics = notifications From 643deafd2b4f67ed22e771bc5d20be4c6a15769d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 15 Oct 2014 07:49:01 +0000 Subject: [PATCH 06/61] Sync 0mq charmhelpers --- .../charmhelpers/contrib/hahelpers/apache.py | 13 +- .../charmhelpers/contrib/hahelpers/cluster.py | 3 +- hooks/charmhelpers/contrib/network/ip.py | 207 ++++++++++++-- .../contrib/openstack/amulet/deployment.py | 46 ++- .../contrib/openstack/amulet/utils.py | 9 +- .../charmhelpers/contrib/openstack/context.py | 267 +++++++++++++----- hooks/charmhelpers/contrib/openstack/ip.py | 2 +- .../contrib/openstack/templates/haproxy.cfg | 29 +- .../templates/openstack_https_frontend | 17 +- .../templates/openstack_https_frontend.conf | 17 +- hooks/charmhelpers/contrib/openstack/utils.py | 31 +- hooks/charmhelpers/core/hookenv.py | 21 +- hooks/charmhelpers/core/host.py | 35 ++- hooks/charmhelpers/core/services/helpers.py | 124 +++++++- hooks/charmhelpers/core/sysctl.py | 34 +++ hooks/charmhelpers/fetch/__init__.py | 24 +- hooks/charmhelpers/fetch/archiveurl.py | 53 +++- 17 files changed, 782 insertions(+), 150 deletions(-) create mode 100644 hooks/charmhelpers/core/sysctl.py diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 8d5fb8ba..6616ffff 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -20,20 +20,27 @@ from charmhelpers.core.hookenv import ( ) -def get_cert(): +def get_cert(cn=None): + # TODO: deal with multiple https endpoints via charm config cert = config_get('ssl_cert') key = config_get('ssl_key') if not (cert and key): log("Inspecting identity-service relations for SSL certificate.", level=INFO) cert = key = None + if cn: + ssl_cert_attr = 'ssl_cert_{}'.format(cn) + ssl_key_attr = 'ssl_key_{}'.format(cn) + else: + ssl_cert_attr = 'ssl_cert' + ssl_key_attr = 'ssl_key' for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): if not cert: - cert = relation_get('ssl_cert', + cert = relation_get(ssl_cert_attr, rid=r_id, unit=unit) if not key: - key = relation_get('ssl_key', + key = relation_get(ssl_key_attr, rid=r_id, unit=unit) return (cert, key) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 7151b1d0..6d972007 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -139,10 +139,9 @@ def https(): return True for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): + # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN rel_state = [ relation_get('https_keystone', rid=r_id, unit=unit), - relation_get('ssl_cert', rid=r_id, unit=unit), - relation_get('ssl_key', rid=r_id, unit=unit), relation_get('ca_cert', rid=r_id, unit=unit), ] # NOTE: works around (LP: #1203241) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 7edbcc48..e62e5655 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,10 +1,16 @@ +import glob +import re +import subprocess import sys from functools import partial +from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - ERROR, log, config, + WARNING, + ERROR, + log ) try: @@ -51,6 +57,8 @@ def get_address_in_network(network, fallback=None, fatal=False): else: if fatal: not_found_error_out() + else: + return None _validate_cidr(network) network = netaddr.IPNetwork(network) @@ -132,7 +140,8 @@ def _get_for_address(address, key): if address.version == 4 and netifaces.AF_INET in addresses: addr = addresses[netifaces.AF_INET][0]['addr'] netmask = addresses[netifaces.AF_INET][0]['netmask'] - cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + network = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + cidr = network.cidr if address in cidr: if key == 'iface': return iface @@ -141,11 +150,14 @@ def _get_for_address(address, key): if address.version == 6 and netifaces.AF_INET6 in addresses: for addr in addresses[netifaces.AF_INET6]: if not addr['addr'].startswith('fe80'): - cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], - addr['netmask'])) + network = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + cidr = network.cidr if address in cidr: if key == 'iface': return iface + elif key == 'netmask' and cidr: + return str(cidr).split('/')[1] else: return addr[key] return None @@ -156,19 +168,182 @@ get_iface_for_address = partial(_get_for_address, key='iface') get_netmask_for_address = partial(_get_for_address, key='netmask') -def get_ipv6_addr(iface="eth0"): +def format_ipv6_addr(address): + """ + IPv6 needs to be wrapped with [] in url link to parse correctly. + """ + if is_ipv6(address): + address = "[%s]" % address + else: + log("Not a valid ipv6 address: %s" % address, level=WARNING) + address = None + + return address + + +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, + fatal=True, exc_list=None): + """ + Return the assigned IP address for a given interface, if any, or []. + """ + # Extract nic if passed /dev/ethX + if '/' in iface: + iface = iface.split('/')[-1] + if not exc_list: + exc_list = [] try: - iface_addrs = netifaces.ifaddresses(iface) - if netifaces.AF_INET6 not in iface_addrs: - raise Exception("Interface '%s' doesn't have an ipv6 address." % iface) + inet_num = getattr(netifaces, inet_type) + except AttributeError: + raise Exception('Unknown inet type ' + str(inet_type)) - addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6] - ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80') - and config('vip') != a['addr']] - if not ipv6_addr: - raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) + interfaces = netifaces.interfaces() + if inc_aliases: + ifaces = [] + for _iface in interfaces: + if iface == _iface or _iface.split(':')[0] == iface: + ifaces.append(_iface) + if fatal and not ifaces: + raise Exception("Invalid interface '%s'" % iface) + ifaces.sort() + else: + if iface not in interfaces: + if fatal: + raise Exception("%s not found " % (iface)) + else: + return [] + else: + ifaces = [iface] - return ipv6_addr[0] + addresses = [] + for netiface in ifaces: + net_info = netifaces.ifaddresses(netiface) + if inet_num in net_info: + for entry in net_info[inet_num]: + if 'addr' in entry and entry['addr'] not in exc_list: + addresses.append(entry['addr']) + if fatal and not addresses: + raise Exception("Interface '%s' doesn't have any %s addresses." % + (iface, inet_type)) + return addresses - except ValueError: - raise ValueError("Invalid interface '%s'" % iface) +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') + + +def get_iface_from_addr(addr): + """Work out on which interface the provided address is configured.""" + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + for inet_type in addresses: + for _addr in addresses[inet_type]: + _addr = _addr['addr'] + # link local + ll_key = re.compile("(.+)%.*") + raw = re.match(ll_key, _addr) + if raw: + _addr = raw.group(1) + if _addr == addr: + log("Address '%s' is configured on iface '%s'" % + (addr, iface)) + return iface + + msg = "Unable to infer net iface on which '%s' is configured" % (addr) + raise Exception(msg) + + +def sniff_iface(f): + """If no iface provided, inject net iface inferred from unit private + address. + """ + def iface_sniffer(*args, **kwargs): + if not kwargs.get('iface', None): + kwargs['iface'] = get_iface_from_addr(unit_get('private-address')) + + return f(*args, **kwargs) + + return iface_sniffer + + +@sniff_iface +def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, + dynamic_only=True): + """Get assigned IPv6 address for a given interface. + + Returns list of addresses found. If no address found, returns empty list. + + If iface is None, we infer the current primary interface by doing a reverse + lookup on the unit private-address. + + We currently only support scope global IPv6 addresses i.e. non-temporary + addresses. If no global IPv6 address is found, return the first one found + in the ipv6 address list. + """ + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', + inc_aliases=inc_aliases, fatal=fatal, + exc_list=exc_list) + + if addresses: + global_addrs = [] + for addr in addresses: + key_scope_link_local = re.compile("^fe80::..(.+)%(.+)") + m = re.match(key_scope_link_local, addr) + if m: + eui_64_mac = m.group(1) + iface = m.group(2) + else: + global_addrs.append(addr) + + if global_addrs: + # Make sure any found global addresses are not temporary + cmd = ['ip', 'addr', 'show', iface] + out = subprocess.check_output(cmd) + if dynamic_only: + key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*") + else: + key = re.compile("inet6 (.+)/[0-9]+ scope global.*") + + addrs = [] + for line in out.split('\n'): + line = line.strip() + m = re.match(key, line) + if m and 'temporary' not in line: + # Return the first valid address we find + for addr in global_addrs: + if m.group(1) == addr: + if not dynamic_only or \ + m.group(1).endswith(eui_64_mac): + addrs.append(addr) + + if addrs: + return addrs + + if fatal: + raise Exception("Interface '%s' doesn't have a scope global " + "non-temporary ipv6 address." % iface) + + return [] + + +def get_bridges(vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of bridges on the system or [] + """ + b_rgex = vnic_dir + '/*/bridge' + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)] + + +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): + """ + Return a list of nics comprising a given bridge on the system or [] + """ + brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_rgex)] + + +def is_bridge_member(nic): + """ + Check if a given nic is a member of a bridge + """ + for bridge in get_bridges(): + if nic in get_bridge_nics(bridge): + return True + return False diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 9179eeb1..3c7f422a 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -10,32 +10,62 @@ class OpenStackAmuletDeployment(AmuletDeployment): that is specifically for use by OpenStack charms. """ - def __init__(self, series=None, openstack=None, source=None): + def __init__(self, series=None, openstack=None, source=None, stable=True): """Initialize the deployment environment.""" super(OpenStackAmuletDeployment, self).__init__(series) self.openstack = openstack self.source = source + self.stable = stable + # Note(coreycb): this needs to be changed when new next branches come + # out. + self.current_next = "trusty" + + def _determine_branch_locations(self, other_services): + """Determine the branch locations for the other services. + + Determine if the local branch being tested is derived from its + stable or next (dev) branch, and based on this, use the corresonding + stable or next branches for the other_services.""" + base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + + if self.stable: + for svc in other_services: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + for svc in other_services: + if svc['name'] in base_charms: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + temp = 'lp:~openstack-charmers/charms/{}/{}/next' + svc['location'] = temp.format(self.current_next, + svc['name']) + return other_services def _add_services(self, this_service, other_services): - """Add services to the deployment and set openstack-origin.""" + """Add services to the deployment and set openstack-origin/source.""" + other_services = self._determine_branch_locations(other_services) + super(OpenStackAmuletDeployment, self)._add_services(this_service, other_services) - name = 0 + services = other_services services.append(this_service) - use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', + 'ceph-osd', 'ceph-radosgw'] if self.openstack: for svc in services: - if svc[name] not in use_source: + if svc['name'] not in use_source: config = {'openstack-origin': self.openstack} - self.d.configure(svc[name], config) + self.d.configure(svc['name'], config) if self.source: for svc in services: - if svc[name] in use_source: + if svc['name'] in use_source: config = {'source': self.source} - self.d.configure(svc[name], config) + self.d.configure(svc['name'], config) def _configure_services(self, configs): """Configure all of the services.""" diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index bd327bdc..0f312b99 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -187,15 +187,16 @@ class OpenStackAmuletUtils(AmuletUtils): f = opener.open("http://download.cirros-cloud.net/version/released") version = f.read().strip() - cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + cirros_img = "cirros-{}-x86_64-disk.img".format(version) + local_path = os.path.join('tests', cirros_img) - if not os.path.exists(cirros_img): + if not os.path.exists(local_path): cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", version, cirros_img) - opener.retrieve(cirros_url, cirros_img) + opener.retrieve(cirros_url, local_path) f.close() - with open(cirros_img) as f: + with open(local_path) as f: image = glance.images.create(name=image_name, is_public=True, disk_format='qcow2', container_format='bare', data=f) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 7c55084d..acd9bca8 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -8,7 +8,6 @@ from subprocess import ( check_call ) - from charmhelpers.fetch import ( apt_install, filter_installed_packages, @@ -16,12 +15,12 @@ from charmhelpers.fetch import ( from charmhelpers.core.hookenv import ( config, + is_relation_made, local_unit, log, relation_get, relation_ids, related_units, - is_relation_made, relation_set, unit_get, unit_private_ip, @@ -29,6 +28,11 @@ from charmhelpers.core.hookenv import ( INFO ) +from charmhelpers.core.host import ( + mkdir, + write_file +) + from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, determine_api_port, @@ -39,6 +43,7 @@ from charmhelpers.contrib.hahelpers.cluster import ( from charmhelpers.contrib.hahelpers.apache import ( get_cert, get_ca_cert, + install_ca_cert, ) from charmhelpers.contrib.openstack.neutron import ( @@ -48,9 +53,13 @@ from charmhelpers.contrib.openstack.neutron import ( from charmhelpers.contrib.network.ip import ( get_address_in_network, get_ipv6_addr, + get_netmask_for_address, + format_ipv6_addr, + is_address_in_network ) from charmhelpers.contrib.openstack.utils import ( + get_host_ip, get_matchmaker_map, ) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -172,8 +181,10 @@ class SharedDBContext(OSContextGenerator): for rid in relation_ids('shared-db'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) + host = rdata.get('db_host') + host = format_ipv6_addr(host) or host ctxt = { - 'database_host': rdata.get('db_host'), + 'database_host': host, 'database': self.database, 'database_user': self.user, 'database_password': rdata.get(password_setting), @@ -249,10 +260,15 @@ class IdentityServiceContext(OSContextGenerator): for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) + serv_host = rdata.get('service_host') + serv_host = format_ipv6_addr(serv_host) or serv_host + auth_host = rdata.get('auth_host') + auth_host = format_ipv6_addr(auth_host) or auth_host + ctxt = { 'service_port': rdata.get('service_port'), - 'service_host': rdata.get('service_host'), - 'auth_host': rdata.get('auth_host'), + 'service_host': serv_host, + 'auth_host': auth_host, 'auth_port': rdata.get('auth_port'), 'admin_tenant_name': rdata.get('service_tenant'), 'admin_user': rdata.get('service_username'), @@ -301,11 +317,13 @@ class AMQPContext(OSContextGenerator): for unit in related_units(rid): if relation_get('clustered', rid=rid, unit=unit): ctxt['clustered'] = True - ctxt['rabbitmq_host'] = relation_get('vip', rid=rid, - unit=unit) + vip = relation_get('vip', rid=rid, unit=unit) + vip = format_ipv6_addr(vip) or vip + ctxt['rabbitmq_host'] = vip else: - ctxt['rabbitmq_host'] = relation_get('private-address', - rid=rid, unit=unit) + host = relation_get('private-address', rid=rid, unit=unit) + host = format_ipv6_addr(host) or host + ctxt['rabbitmq_host'] = host ctxt.update({ 'rabbitmq_user': username, 'rabbitmq_password': relation_get('password', rid=rid, @@ -344,8 +362,9 @@ class AMQPContext(OSContextGenerator): and len(related_units(rid)) > 1: rabbitmq_hosts = [] for unit in related_units(rid): - rabbitmq_hosts.append(relation_get('private-address', - rid=rid, unit=unit)) + host = relation_get('private-address', rid=rid, unit=unit) + host = format_ipv6_addr(host) or host + rabbitmq_hosts.append(host) ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) if not context_complete(ctxt): return {} @@ -374,6 +393,7 @@ class CephContext(OSContextGenerator): ceph_addr = \ relation_get('ceph-public-address', rid=rid, unit=unit) or \ relation_get('private-address', rid=rid, unit=unit) + ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr mon_hosts.append(ceph_addr) ctxt = { @@ -394,6 +414,9 @@ class CephContext(OSContextGenerator): return ctxt +ADDRESS_TYPES = ['admin', 'internal', 'public'] + + class HAProxyContext(OSContextGenerator): interfaces = ['cluster'] @@ -406,25 +429,63 @@ class HAProxyContext(OSContextGenerator): if not relation_ids('cluster'): return {} - cluster_hosts = {} l_unit = local_unit().replace('/', '-') - if config('prefer-ipv6'): - addr = get_ipv6_addr() - else: - addr = unit_get('private-address') - cluster_hosts[l_unit] = get_address_in_network(config('os-internal-network'), - addr) - for rid in relation_ids('cluster'): - for unit in related_units(rid): - _unit = unit.replace('/', '-') - addr = relation_get('private-address', rid=rid, unit=unit) - cluster_hosts[_unit] = addr + if config('prefer-ipv6'): + addr = get_ipv6_addr(exc_list=[config('vip')])[0] + else: + addr = get_host_ip(unit_get('private-address')) + + cluster_hosts = {} + + # NOTE(jamespage): build out map of configured network endpoints + # and associated backends + for addr_type in ADDRESS_TYPES: + laddr = get_address_in_network( + config('os-{}-network'.format(addr_type))) + if laddr: + cluster_hosts[laddr] = {} + cluster_hosts[laddr]['network'] = "{}/{}".format( + laddr, + get_netmask_for_address(laddr) + ) + cluster_hosts[laddr]['backends'] = {} + cluster_hosts[laddr]['backends'][l_unit] = laddr + for rid in relation_ids('cluster'): + for unit in related_units(rid): + _unit = unit.replace('/', '-') + _laddr = relation_get('{}-address'.format(addr_type), + rid=rid, unit=unit) + if _laddr: + cluster_hosts[laddr]['backends'][_unit] = _laddr + + # NOTE(jamespage) no split configurations found, just use + # private addresses + if not cluster_hosts: + cluster_hosts[addr] = {} + cluster_hosts[addr]['network'] = "{}/{}".format( + addr, + get_netmask_for_address(addr) + ) + cluster_hosts[addr]['backends'] = {} + cluster_hosts[addr]['backends'][l_unit] = addr + for rid in relation_ids('cluster'): + for unit in related_units(rid): + _unit = unit.replace('/', '-') + _laddr = relation_get('private-address', + rid=rid, unit=unit) + if _laddr: + cluster_hosts[addr]['backends'][_unit] = _laddr ctxt = { - 'units': cluster_hosts, + 'frontends': cluster_hosts, } + if config('haproxy-server-timeout'): + ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout') + if config('haproxy-client-timeout'): + ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') + if config('prefer-ipv6'): ctxt['local_host'] = 'ip6-localhost' ctxt['haproxy_host'] = '::' @@ -434,12 +495,13 @@ class HAProxyContext(OSContextGenerator): ctxt['haproxy_host'] = '0.0.0.0' ctxt['stat_port'] = ':8888' - if len(cluster_hosts.keys()) > 1: - # Enable haproxy when we have enough peers. - log('Ensuring haproxy enabled in /etc/default/haproxy.') - with open('/etc/default/haproxy', 'w') as out: - out.write('ENABLED=1\n') - return ctxt + for frontend in cluster_hosts: + if len(cluster_hosts[frontend]['backends']) > 1: + # Enable haproxy when we have enough peers. + log('Ensuring haproxy enabled in /etc/default/haproxy.') + with open('/etc/default/haproxy', 'w') as out: + out.write('ENABLED=1\n') + return ctxt log('HAProxy context is incomplete, this unit has no peers.') return {} @@ -494,22 +556,36 @@ class ApacheSSLContext(OSContextGenerator): cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http'] check_call(cmd) - def configure_cert(self): - if not os.path.isdir('/etc/apache2/ssl'): - os.mkdir('/etc/apache2/ssl') + def configure_cert(self, cn=None): ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) - if not os.path.isdir(ssl_dir): - os.mkdir(ssl_dir) - cert, key = get_cert() - with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out: - cert_out.write(b64decode(cert)) - with open(os.path.join(ssl_dir, 'key'), 'w') as key_out: - key_out.write(b64decode(key)) + mkdir(path=ssl_dir) + cert, key = get_cert(cn) + if cn: + cert_filename = 'cert_{}'.format(cn) + key_filename = 'key_{}'.format(cn) + else: + cert_filename = 'cert' + key_filename = 'key' + write_file(path=os.path.join(ssl_dir, cert_filename), + content=b64decode(cert)) + write_file(path=os.path.join(ssl_dir, key_filename), + content=b64decode(key)) + + def configure_ca(self): ca_cert = get_ca_cert() if ca_cert: - with open(CA_CERT_PATH, 'w') as ca_out: - ca_out.write(b64decode(ca_cert)) - check_call(['update-ca-certificates']) + install_ca_cert(b64decode(ca_cert)) + + def canonical_names(self): + '''Figure out which canonical names clients will access this service''' + cns = [] + for r_id in relation_ids('identity-service'): + for unit in related_units(r_id): + rdata = relation_get(rid=r_id, unit=unit) + for k in rdata: + if k.startswith('ssl_key_'): + cns.append(k.lstrip('ssl_key_')) + return list(set(cns)) def __call__(self): if isinstance(self.external_ports, basestring): @@ -517,21 +593,47 @@ class ApacheSSLContext(OSContextGenerator): if (not self.external_ports or not https()): return {} - self.configure_cert() + self.configure_ca() self.enable_modules() ctxt = { 'namespace': self.service_namespace, - 'private_address': unit_get('private-address'), - 'endpoints': [] + 'endpoints': [], + 'ext_ports': [] } - if is_clustered(): - ctxt['private_address'] = config('vip') - for api_port in self.external_ports: - ext_port = determine_apache_port(api_port) - int_port = determine_api_port(api_port) - portmap = (int(ext_port), int(int_port)) - ctxt['endpoints'].append(portmap) + + for cn in self.canonical_names(): + self.configure_cert(cn) + + addresses = [] + vips = [] + if config('vip'): + vips = config('vip').split() + + for network_type in ['os-internal-network', + 'os-admin-network', + 'os-public-network']: + address = get_address_in_network(config(network_type), + unit_get('private-address')) + if len(vips) > 0 and is_clustered(): + for vip in vips: + if is_address_in_network(config(network_type), + vip): + addresses.append((address, vip)) + break + elif is_clustered(): + addresses.append((address, config('vip'))) + else: + addresses.append((address, address)) + + for address, endpoint in set(addresses): + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port) + int_port = determine_api_port(api_port) + portmap = (address, endpoint, int(ext_port), int(int_port)) + ctxt['endpoints'].append(portmap) + ctxt['ext_ports'].append(int(ext_port)) + ctxt['ext_ports'] = list(set(ctxt['ext_ports'])) return ctxt @@ -661,22 +763,22 @@ class NeutronContext(OSContextGenerator): class OSConfigFlagContext(OSContextGenerator): - """ - Responsible for adding user-defined config-flags in charm config to a - template context. + """ + Responsible for adding user-defined config-flags in charm config to a + template context. - NOTE: the value of config-flags may be a comma-separated list of - key=value pairs and some Openstack config files support - comma-separated lists as values. - """ + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ - def __call__(self): - config_flags = config('config-flags') - if not config_flags: - return {} + def __call__(self): + config_flags = config('config-flags') + if not config_flags: + return {} - flags = config_flags_parser(config_flags) - return {'user_config_flags': flags} + flags = config_flags_parser(config_flags) + return {'user_config_flags': flags} class SubordinateConfigContext(OSContextGenerator): @@ -793,6 +895,38 @@ class SyslogContext(OSContextGenerator): return ctxt +class BindHostContext(OSContextGenerator): + + def __call__(self): + if config('prefer-ipv6'): + return { + 'bind_host': '::' + } + else: + return { + 'bind_host': '0.0.0.0' + } + + +class WorkerConfigContext(OSContextGenerator): + + @property + def num_cpus(self): + try: + from psutil import NUM_CPUS + except ImportError: + apt_install('python-psutil', fatal=True) + from psutil import NUM_CPUS + return NUM_CPUS + + def __call__(self): + multiplier = config('worker-multiplier') or 1 + ctxt = { + "workers": self.num_cpus * multiplier + } + return ctxt + + class ZeroMQContext(OSContextGenerator): interfaces = ['zeromq-configuration'] @@ -805,6 +939,7 @@ class ZeroMQContext(OSContextGenerator): ctxt['zmq_host'] = relation_get('host', unit, rid) return ctxt + class NotificationDriverContext(OSContextGenerator): def __init__(self, zmq_relation='zeromq-configuration', amqp_relation='amqp'): diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index affe8cd1..bc84fc45 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -66,7 +66,7 @@ def resolve_address(endpoint_type=PUBLIC): resolved_address = vip else: if config('prefer-ipv6'): - fallback_addr = get_ipv6_addr() + fallback_addr = get_ipv6_addr(exc_list=[config('vip')])[0] else: fallback_addr = unit_get(_address_map[endpoint_type]['fallback']) resolved_address = get_address_in_network( diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index ce0e2738..19c9b856 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -14,8 +14,17 @@ defaults retries 3 timeout queue 1000 timeout connect 1000 +{% if haproxy_client_timeout -%} + timeout client {{ haproxy_client_timeout }} +{% else -%} timeout client 30000 +{% endif -%} + +{% if haproxy_server_timeout -%} + timeout server {{ haproxy_server_timeout }} +{% else -%} timeout server 30000 +{% endif -%} listen stats {{ stat_port }} mode http @@ -25,17 +34,21 @@ listen stats {{ stat_port }} stats uri / stats auth admin:password -{% if units -%} +{% if frontends -%} {% for service, ports in service_ports.iteritems() -%} -listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }} - balance roundrobin - {% for unit, address in units.iteritems() -%} - server {{ unit }} {{ address }}:{{ ports[1] }} check +frontend tcp-in_{{ service }} + bind *:{{ ports[0] }} + bind :::{{ ports[0] }} + {% for frontend in frontends -%} + acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} + use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} {% endfor %} -listen {{ service }}_ipv6 :::{{ ports[0] }} - balance roundrobin - {% for unit, address in units.iteritems() -%} +{% for frontend in frontends -%} +backend {{ service }}_{{ frontend }} + balance leastconn + {% for unit, address in frontends[frontend]['backends'].iteritems() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} {% endfor -%} +{% endfor -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend index e02dc751..ce28fa3f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend @@ -1,16 +1,18 @@ {% if endpoints -%} -{% for ext, int in endpoints -%} -Listen {{ ext }} -NameVirtualHost *:{{ ext }} - - ServerName {{ private_address }} +{% for ext_port in ext_ports -%} +Listen {{ ext_port }} +{% endfor -%} +{% for address, endpoint, ext, int in endpoints -%} + + ServerName {{ endpoint }} SSLEngine on - SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert - SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ ProxyPassReverse / http://localhost:{{ int }}/ ProxyPreserveHost on +{% endfor -%} Order deny,allow Allow from all @@ -19,5 +21,4 @@ NameVirtualHost *:{{ ext }} Order allow,deny Allow from all -{% endfor -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf index e02dc751..ce28fa3f 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf +++ b/hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -1,16 +1,18 @@ {% if endpoints -%} -{% for ext, int in endpoints -%} -Listen {{ ext }} -NameVirtualHost *:{{ ext }} - - ServerName {{ private_address }} +{% for ext_port in ext_ports -%} +Listen {{ ext_port }} +{% endfor -%} +{% for address, endpoint, ext, int in endpoints -%} + + ServerName {{ endpoint }} SSLEngine on - SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert - SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} ProxyPass / http://localhost:{{ int }}/ ProxyPassReverse / http://localhost:{{ int }}/ ProxyPreserveHost on +{% endfor -%} Order deny,allow Allow from all @@ -19,5 +21,4 @@ NameVirtualHost *:{{ ext }} Order allow,deny Allow from all -{% endfor -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 4bcd3cff..90bdad61 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -3,19 +3,20 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict +import subprocess import json import os import socket -import subprocess import sys from charmhelpers.core.hookenv import ( config, log as juju_log, charm_dir, - is_relation_made, ERROR, - INFO + INFO, + relation_ids, + relation_set ) from charmhelpers.contrib.storage.linux.lvm import ( @@ -24,6 +25,10 @@ from charmhelpers.contrib.storage.linux.lvm import ( remove_lvm_physical_volume, ) +from charmhelpers.contrib.network.ip import ( + get_ipv6_addr +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk @@ -73,6 +78,8 @@ SWIFT_CODENAMES = OrderedDict([ ('1.12.0', 'icehouse'), ('1.11.0', 'icehouse'), ('2.0.0', 'juno'), + ('2.1.0', 'juno'), + ('2.2.0', 'juno'), ]) DEFAULT_LOOPBACK_SIZE = '5G' @@ -467,3 +474,21 @@ def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): with open(mm_file, 'r') as f: mm_map = json.load(f) return mm_map + + +def sync_db_with_multi_ipv6_addresses(database, database_user, + relation_prefix=None): + hosts = get_ipv6_addr(dynamic_only=False) + + kwargs = {'database': database, + 'username': database_user, + 'hostname': json.dumps(hosts)} + + if relation_prefix: + keys = kwargs.keys() + for key in keys: + kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key] + del kwargs[key] + + for rid in relation_ids('shared-db'): + relation_set(relation_id=rid, **kwargs) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index f396e03a..af8fe2db 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -203,6 +203,17 @@ class Config(dict): if os.path.exists(self.path): self.load_previous() + def __getitem__(self, key): + """For regular dict lookups, check the current juju config first, + then the previous (saved) copy. This ensures that user-saved values + will be returned by a dict lookup. + + """ + try: + return dict.__getitem__(self, key) + except KeyError: + return (self._prev_dict or {})[key] + def load_previous(self, path=None): """Load previous copy of config from disk. @@ -475,9 +486,10 @@ class Hooks(object): hooks.execute(sys.argv) """ - def __init__(self): + def __init__(self, config_save=True): super(Hooks, self).__init__() self._hooks = {} + self._config_save = config_save def register(self, name, function): """Register a hook""" @@ -488,9 +500,10 @@ class Hooks(object): hook_name = os.path.basename(args[0]) if hook_name in self._hooks: self._hooks[hook_name]() - cfg = config() - if cfg.implicit_save: - cfg.save() + if self._config_save: + cfg = config() + if cfg.implicit_save: + cfg.save() else: raise UnregisteredHookError(hook_name) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b85b0280..d7ce1e4c 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -68,8 +68,8 @@ def service_available(service_name): """Determine whether a system service is available""" try: subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - return False + except subprocess.CalledProcessError as e: + return 'unrecognized service' not in e.output else: return True @@ -209,10 +209,15 @@ def mounts(): return system_mounts -def file_hash(path): - """Generate a md5 hash of the contents of 'path' or None if not found """ +def file_hash(path, hash_type='md5'): + """ + Generate a hash checksum of the contents of 'path' or None if not found. + + :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ if os.path.exists(path): - h = hashlib.md5() + h = getattr(hashlib, hash_type)() with open(path, 'r') as source: h.update(source.read()) # IGNORE:E1101 - it does have update return h.hexdigest() @@ -220,6 +225,26 @@ def file_hash(path): return None +def check_hash(path, checksum, hash_type='md5'): + """ + Validate a file using a cryptographic checksum. + + :param str checksum: Value of the checksum used to validate the file. + :param str hash_type: Hash algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + :raises ChecksumError: If the file fails the checksum + + """ + actual_checksum = file_hash(path, hash_type) + if checksum != actual_checksum: + raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) + + +class ChecksumError(ValueError): + pass + + def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 4b90589b..7067b94b 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -1,3 +1,5 @@ +import os +import yaml from charmhelpers.core import hookenv from charmhelpers.core import templating @@ -19,15 +21,21 @@ class RelationContext(dict): the `name` attribute that are complete will used to populate the dictionary values (see `get_data`, below). - The generated context will be namespaced under the interface type, to prevent - potential naming conflicts. + The generated context will be namespaced under the relation :attr:`name`, + to prevent potential naming conflicts. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` """ name = None interface = None required_keys = [] - def __init__(self, *args, **kwargs): - super(RelationContext, self).__init__(*args, **kwargs) + def __init__(self, name=None, additional_required_keys=None): + if name is not None: + self.name = name + if additional_required_keys is not None: + self.required_keys.extend(additional_required_keys) self.get_data() def __bool__(self): @@ -101,9 +109,115 @@ class RelationContext(dict): return {} +class MysqlRelation(RelationContext): + """ + Relation context for the `mysql` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'db' + interface = 'mysql' + required_keys = ['host', 'user', 'password', 'database'] + + +class HttpRelation(RelationContext): + """ + Relation context for the `http` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'website' + interface = 'http' + required_keys = ['host', 'port'] + + def provide_data(self): + return { + 'host': hookenv.unit_get('private-address'), + 'port': 80, + } + + +class RequiredConfig(dict): + """ + Data context that loads config options with one or more mandatory options. + + Once the required options have been changed from their default values, all + config options will be available, namespaced under `config` to prevent + potential naming conflicts (for example, between a config option and a + relation property). + + :param list *args: List of options that must be changed from their default values. + """ + + def __init__(self, *args): + self.required_options = args + self['config'] = hookenv.config() + with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: + self.config = yaml.load(fp).get('options', {}) + + def __bool__(self): + for option in self.required_options: + if option not in self['config']: + return False + current_value = self['config'][option] + default_value = self.config[option].get('default') + if current_value == default_value: + return False + if current_value in (None, '') and default_value in (None, ''): + return False + return True + + def __nonzero__(self): + return self.__bool__() + + +class StoredContext(dict): + """ + A data context that always returns the data that it was first created with. + + This is useful to do a one-time generation of things like passwords, that + will thereafter use the same value that was originally generated, instead + of generating a new value each time it is run. + """ + def __init__(self, file_name, config_data): + """ + If the file exists, populate `self` with the data from the file. + Otherwise, populate with the given data and persist it to the file. + """ + if os.path.exists(file_name): + self.update(self.read_context(file_name)) + else: + self.store_context(file_name, config_data) + self.update(config_data) + + def store_context(self, file_name, config_data): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'w') as file_stream: + os.fchmod(file_stream.fileno(), 0600) + yaml.dump(config_data, file_stream) + + def read_context(self, file_name): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'r') as file_stream: + data = yaml.load(file_stream) + if not data: + raise OSError("%s is empty" % file_name) + return data + + class TemplateCallback(ManagerCallback): """ - Callback class that will render a template, for use as a ready action. + Callback class that will render a Jinja2 template, for use as a ready action. + + :param str source: The template source file, relative to `$CHARM_DIR/templates` + :param str target: The target to write the rendered template to + :param str owner: The owner of the rendered file + :param str group: The group of the rendered file + :param int perms: The permissions of the rendered file """ def __init__(self, source, target, owner='root', group='root', perms=0444): self.source = source diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py new file mode 100644 index 00000000..0f299630 --- /dev/null +++ b/hooks/charmhelpers/core/sysctl.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__author__ = 'Jorge Niedbalski R. ' + +import yaml + +from subprocess import check_call + +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) + + +def create(sysctl_dict, sysctl_file): + """Creates a sysctl.conf file from a YAML associative array + + :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } + :type sysctl_dict: dict + :param sysctl_file: path to the sysctl file to be saved + :type sysctl_file: str or unicode + :returns: None + """ + sysctl_dict = yaml.load(sysctl_dict) + + with open(sysctl_file, "w") as fd: + for key, value in sysctl_dict.items(): + fd.write("{}={}\n".format(key, value)) + + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), + level=DEBUG) + + check_call(["sysctl", "-p", sysctl_file]) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 8e9d3804..32a673d6 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -208,7 +208,8 @@ def add_source(source, key=None): """Add a package source to this system. @param source: a URL or sources.list entry, as supported by - add-apt-repository(1). Examples: + add-apt-repository(1). Examples:: + ppa:charmers/example deb https://stub:key@private.example.com/ubuntu trusty main @@ -311,22 +312,35 @@ def configure_sources(update=False, apt_update(fatal=True) -def install_remote(source): +def install_remote(source, *args, **kwargs): """ Install a file tree from a remote source The specified source should be a url of the form: scheme://[host]/path[#[option=value][&...]] - Schemes supported are based on this modules submodules - Options supported are submodule-specific""" + Schemes supported are based on this modules submodules. + Options supported are submodule-specific. + Additional arguments are passed through to the submodule. + + For example:: + + dest = install_remote('http://example.com/archive.tgz', + checksum='deadbeef', + hash_type='sha1') + + This will download `archive.tgz`, validate it using SHA1 and, if + the file is ok, extract it and return the directory in which it + was extracted. If the checksum fails, it will raise + :class:`charmhelpers.core.host.ChecksumError`. + """ # We ONLY check for True here because can_handle may return a string # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None for handler in handlers: try: - installed_to = handler.install(source) + installed_to = handler.install(source, *args, **kwargs) except UnhandledSource: pass if not installed_to: diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index 87e7071a..8c045650 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -1,6 +1,8 @@ import os import urllib2 +from urllib import urlretrieve import urlparse +import hashlib from charmhelpers.fetch import ( BaseFetchHandler, @@ -10,11 +12,19 @@ from charmhelpers.payload.archive import ( get_archive_handler, extract, ) -from charmhelpers.core.host import mkdir +from charmhelpers.core.host import mkdir, check_hash class ArchiveUrlFetchHandler(BaseFetchHandler): - """Handler for archives via generic URLs""" + """ + Handler to download archive files from arbitrary URLs. + + Can fetch from http, https, ftp, and file URLs. + + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. + + Installs the contents of the archive in $CHARM_DIR/fetched/. + """ def can_handle(self, source): url_parts = self.parse_url(source) if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): @@ -24,6 +34,12 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): return False def download(self, source, dest): + """ + Download an archive file. + + :param str source: URL pointing to an archive file. + :param str dest: Local path location to download archive file to. + """ # propogate all exceptions # URLError, OSError, etc proto, netloc, path, params, query, fragment = urlparse.urlparse(source) @@ -48,7 +64,30 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): os.unlink(dest) raise e - def install(self, source): + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + tempfile, headers = urlretrieve(url) + check_hash(tempfile, hashsum, validate) + return tempfile + + def install(self, source, dest=None, checksum=None, hash_type='sha1'): + """ + Download and install an archive file, with optional checksum validation. + + The checksum can also be given on the `source` URL's fragment. + For example:: + + handler.install('http://example.com/file.tgz#sha1=deadbeef') + + :param str source: URL pointing to an archive file. + :param str dest: Local destination path to install to. If not given, + installs to `$CHARM_DIR/archives/archive_file_name`. + :param str checksum: If given, validate the archive file after download. + :param str hash_type: Algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + + """ url_parts = self.parse_url(source) dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') if not os.path.exists(dest_dir): @@ -60,4 +99,10 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): raise UnhandledSource(e.reason) except OSError as e: raise UnhandledSource(e.strerror) - return extract(dld_file) + options = urlparse.parse_qs(url_parts.fragment) + for key, value in options.items(): + if key in hashlib.algorithms: + check_hash(dld_file, value, key) + if checksum: + check_hash(dld_file, checksum, hash_type) + return extract(dld_file, dest) From e252e581d737662ce2e67773e1d97a97ed35ae1d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 15 Oct 2014 08:56:43 +0000 Subject: [PATCH 07/61] Add support for 0mq l2pop topic --- hooks/neutron_ovs_context.py | 14 +++++++------- hooks/neutron_ovs_utils.py | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 9178a35c..2d73c262 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -18,7 +18,7 @@ OVS_BRIDGE = 'br-int' DATA_BRIDGE = 'br-data' -def _neutron_api_settings(): +def neutron_api_settings(): ''' Inspects current neutron-plugin relation ''' @@ -53,9 +53,9 @@ class OVSPluginContext(context.NeutronContext): return 'neutron' @property - def neutron_security_groups(self): - neutron_api_settings = _neutron_api_settings() - return neutron_api_settings['neutron_security_groups'] + def nutron_security_groups(self): + napi_settings = neutron_api_settings() + return napi_settings['neutron_security_groups'] def get_data_port(self): data_ports = config('data-port') @@ -97,11 +97,11 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['local_ip'] = \ get_address_in_network(config('os-data-network'), get_host_ip(unit_get('private-address'))) - neutron_api_settings = _neutron_api_settings() + napi_settings = neutron_api_settings() ovs_ctxt['neutron_security_groups'] = self.neutron_security_groups - ovs_ctxt['l2_population'] = neutron_api_settings['l2_population'] + ovs_ctxt['l2_population'] = napi_settings['l2_population'] ovs_ctxt['overlay_network_type'] = \ - neutron_api_settings['overlay_network_type'] + napi_settings['overlay_network_type'] # TODO: We need to sort out the syslog and debug/verbose options as a # general context helper ovs_ctxt['use_syslog'] = conf['use-syslog'] diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index e1cb16fd..c43019fe 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -67,5 +67,7 @@ def get_topics(): topics.append('q-agent-notifier-tunnel-update') topics.append('q-agent-notifier-security_group-update') topics.append('q-agent-notifier-dvr-update') + if neutron_ovs_context.neutron_api_settings()['l2_population']: + topics.append('q-agent-notifier-l2pop') return topics From 20dd760ef07436349fdef80de2015ca78e71842c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 15 Oct 2014 09:06:01 +0000 Subject: [PATCH 08/61] Fix topic name --- hooks/neutron_ovs_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index c43019fe..abc4281d 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -68,6 +68,6 @@ def get_topics(): topics.append('q-agent-notifier-security_group-update') topics.append('q-agent-notifier-dvr-update') if neutron_ovs_context.neutron_api_settings()['l2_population']: - topics.append('q-agent-notifier-l2pop') + topics.append('q-agent-notifier-l2population-update') return topics From 26cb60a9d99f9b1e1f09cbf1df07214fd5643363 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 20 Oct 2014 09:58:41 +0000 Subject: [PATCH 09/61] Point charm-helpers sync back at trunk and sync --- charm-helpers-sync.yaml | 2 +- hooks/charmhelpers/contrib/openstack/utils.py | 16 ++++++++++++++++ hooks/charmhelpers/contrib/storage/linux/ceph.py | 7 ++++--- hooks/charmhelpers/core/host.py | 9 ++++++++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 30a27fb0..8af0007c 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:~openstack-charmers/charm-helpers/0mq +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 90bdad61..ae24fb91 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -2,6 +2,7 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict +from functools import wraps import subprocess import json @@ -492,3 +493,18 @@ def sync_db_with_multi_ipv6_addresses(database, database_user, for rid in relation_ids('shared-db'): relation_set(relation_id=rid, **kwargs) + + +def os_requires_version(ostack_release, pkg): + """ + Decorator for hook to specify minimum supported release + """ + def wrap(f): + @wraps(f) + def wrapped_f(*args): + if os_release(pkg) < ostack_release: + raise Exception("This hook is not supported on releases" + " before %s" % ostack_release) + f(*args) + return wrapped_f + return wrap diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 768438a4..598ec263 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -113,7 +113,7 @@ def get_osds(service): return None -def create_pool(service, name, replicas=2): +def create_pool(service, name, replicas=3): ''' Create a new RADOS pool ''' if pool_exists(service, name): log("Ceph pool {} already exists, skipping creation".format(name), @@ -300,7 +300,8 @@ def copy_files(src, dst, symlinks=False, ignore=None): def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, - blk_device, fstype, system_services=[]): + blk_device, fstype, system_services=[], + replicas=3): """ NOTE: This function must only be called from a single service unit for the same rbd_img otherwise data loss will occur. @@ -317,7 +318,7 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, # Ensure pool, RBD image, RBD mappings are in place. if not pool_exists(service, pool): log('ceph: Creating new pool {}.'.format(pool)) - create_pool(service, pool) + create_pool(service, pool, replicas=replicas) if not rbd_exists(service, pool, rbd_img): log('ceph: Creating RBD image ({}).'.format(rbd_img)) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index d7ce1e4c..8a91a5d6 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -6,6 +6,7 @@ # Matthew Wedgwood import os +import re import pwd import grp import random @@ -317,7 +318,13 @@ def list_nics(nic_type): ip_output = (line for line in ip_output if line) for line in ip_output: if line.split()[1].startswith(int_type): - interfaces.append(line.split()[1].replace(":", "")) + matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) + if matched: + interface = matched.groups()[0] + else: + interface = line.split()[1].replace(":", "") + interfaces.append(interface) + return interfaces From 7328d593a73bf2712a749280f03a3cc4a3475b71 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 20 Oct 2014 10:15:19 +0000 Subject: [PATCH 10/61] Use os_requires_version to ensure that zmq is only used with >= juno --- hooks/neutron_ovs_hooks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 75841c9b..e78a8daf 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -19,6 +19,10 @@ from charmhelpers.fetch import ( apt_install, apt_update ) +from charmhelpers.contrib.openstack.utils import ( + os_requires_version, +) + from neutron_ovs_utils import ( determine_packages, get_topics, @@ -66,6 +70,7 @@ def amqp_changed(): @hooks.hook('zeromq-configuration-relation-joined') +@os_requires_version('juno', 'neutron-common') def zeromq_configuration_relation_joined(relid=None): relation_set(relation_id=relid, topics=" ".join(get_topics()), From 48c2f8387e063700037df58517bce498475731fa Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 20 Oct 2014 11:20:06 +0000 Subject: [PATCH 11/61] Sync charmhelpers --- hooks/charmhelpers/contrib/openstack/context.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index acd9bca8..f65658a7 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -60,7 +60,6 @@ from charmhelpers.contrib.network.ip import ( from charmhelpers.contrib.openstack.utils import ( get_host_ip, - get_matchmaker_map, ) CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -951,12 +950,8 @@ class NotificationDriverContext(OSContextGenerator): def __call__(self): ctxt = { - 'notifications': "False", + 'notifications': 'False', } - if is_relation_made(self.zmq_relation): - matchmaker_data = get_matchmaker_map() - if 'notifications-info' in matchmaker_data: - ctxt['notifications'] = "True" - elif is_relation_made(self.amqp_relation): + if is_relation_made(self.amqp_relation): ctxt['notifications'] = "True" return ctxt From c076fee5223e578f2fb8fbb1d4ebc488869f3d1c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 20 Oct 2014 14:07:59 +0000 Subject: [PATCH 12/61] Fix lint and unit tests --- hooks/neutron_ovs_context.py | 2 +- hooks/neutron_ovs_utils.py | 1 - unit_tests/test_neutron_ovs_hooks.py | 5 +++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 2d73c262..cf8f27e4 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -53,7 +53,7 @@ class OVSPluginContext(context.NeutronContext): return 'neutron' @property - def nutron_security_groups(self): + def neutron_security_groups(self): napi_settings = neutron_api_settings() return napi_settings['neutron_security_groups'] diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index abc4281d..51b6ae73 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -70,4 +70,3 @@ def get_topics(): if neutron_ovs_context.neutron_api_settings()['l2_population']: topics.append('q-agent-notifier-l2population-update') return topics - diff --git a/unit_tests/test_neutron_ovs_hooks.py b/unit_tests/test_neutron_ovs_hooks.py index a22f8063..12ecc451 100644 --- a/unit_tests/test_neutron_ovs_hooks.py +++ b/unit_tests/test_neutron_ovs_hooks.py @@ -25,6 +25,7 @@ TO_PATCH = [ 'CONFIGS', 'determine_packages', 'log', + 'relation_ids', 'relation_set', ] NEUTRON_CONF_DIR = "/etc/neutron" @@ -38,6 +39,7 @@ class NeutronOVSHooksTests(CharmTestCase): super(NeutronOVSHooksTests, self).setUp(hooks, TO_PATCH) self.config.side_effect = self.test_config.get + hooks.hooks._config_save = False def _call_hook(self, hookname): hooks.hooks.execute([ @@ -53,8 +55,11 @@ class NeutronOVSHooksTests(CharmTestCase): ]) def test_config_changed(self): + self.relation_ids.return_value = ['relid'] + _zmq_joined = self.patch('zeromq_configuration_relation_joined') self._call_hook('config-changed') self.assertTrue(self.CONFIGS.write_all.called) + self.assertTrue(_zmq_joined.called_with('relid')) def test_amqp_joined(self): self._call_hook('amqp-relation-joined') From 5ad7a1e4e883e69274608719a51e1a3ae8ecf994 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 13 Jan 2015 09:20:58 +0000 Subject: [PATCH 13/61] Resync helpers --- .../charmhelpers/contrib/hahelpers/cluster.py | 38 ++++++++++------ .../charmhelpers/contrib/openstack/context.py | 6 ++- .../charmhelpers/contrib/openstack/neutron.py | 10 ++++- .../contrib/openstack/templates/haproxy.cfg | 2 + hooks/charmhelpers/contrib/openstack/utils.py | 6 +++ .../contrib/storage/linux/ceph.py | 43 +++++++++++++++++++ hooks/charmhelpers/core/host.py | 11 +++-- hooks/charmhelpers/fetch/__init__.py | 9 +++- 8 files changed, 104 insertions(+), 21 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 52ce4b7c..912b2fe3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -13,6 +13,7 @@ clustering-related helpers. import subprocess import os + from socket import gethostname as get_unit_hostname import six @@ -28,12 +29,19 @@ from charmhelpers.core.hookenv import ( WARNING, unit_get, ) +from charmhelpers.core.decorators import ( + retry_on_exception, +) class HAIncompleteConfig(Exception): pass +class CRMResourceNotFound(Exception): + pass + + def is_elected_leader(resource): """ Returns True if the charm executing this is the elected cluster leader. @@ -68,24 +76,30 @@ def is_clustered(): return False -def is_crm_leader(resource): +@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound) +def is_crm_leader(resource, retry=False): """ Returns True if the charm calling this is the elected corosync leader, as returned by calling the external "crm" command. + + We allow this operation to be retried to avoid the possibility of getting a + false negative. See LP #1396246 for more info. """ - cmd = [ - "crm", "resource", - "show", resource - ] + cmd = ['crm', 'resource', 'show', resource] try: - status = subprocess.check_output(cmd).decode('UTF-8') + status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + if not isinstance(status, six.text_type): + status = six.text_type(status, "utf-8") except subprocess.CalledProcessError: - return False - else: - if get_unit_hostname() in status: - return True - else: - return False + status = None + + if status and get_unit_hostname() in status: + return True + + if status and "resource %s is NOT running" % (resource) in status: + raise CRMResourceNotFound("CRM resource %s not found" % (resource)) + + return False def is_leader(resource): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index eb108910..8ab61bf0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -491,6 +491,7 @@ class HAProxyContext(OSContextGenerator): ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') if config('prefer-ipv6'): + ctxt['ipv6'] = True ctxt['local_host'] = 'ip6-localhost' ctxt['haproxy_host'] = '::' ctxt['stat_port'] = ':::8888' @@ -662,8 +663,9 @@ class ApacheSSLContext(OSContextGenerator): addresses = self.get_network_addresses() for address, endpoint in sorted(set(addresses)): for api_port in self.external_ports: - ext_port = determine_apache_port(api_port) - int_port = determine_api_port(api_port) + ext_port = determine_apache_port(api_port, + singlenode_mode=True) + int_port = determine_api_port(api_port, singlenode_mode=True) portmap = (address, endpoint, int(ext_port), int(int_port)) ctxt['endpoints'].append(portmap) ctxt['ext_ports'].append(int(ext_port)) diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 1446f637..095cc24b 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -152,9 +152,15 @@ def neutron_plugins(): database=config('neutron-database'), relation_prefix='neutron', ssl_dir=NEUTRON_CONF_DIR)], - 'services': ['calico-compute', 'bird', 'neutron-dhcp-agent'], + 'services': ['calico-felix', + 'bird', + 'neutron-dhcp-agent', + 'nova-api-metadata'], 'packages': [[headers_package()] + determine_dkms_package(), - ['calico-compute', 'bird', 'neutron-dhcp-agent']], + ['calico-compute', + 'bird', + 'neutron-dhcp-agent', + 'nova-api-metadata']], 'server_packages': ['neutron-server', 'calico-control'], 'server_services': ['neutron-server'] } diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 0229f9d4..9ae1efb9 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -38,7 +38,9 @@ listen stats {{ stat_port }} {% for service, ports in service_ports.items() -%} frontend tcp-in_{{ service }} bind *:{{ ports[0] }} + {% if ipv6 -%} bind :::{{ ports[0] }} + {% endif -%} {% for frontend in frontends -%} acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 44179679..ddd40ce5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,6 +53,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('saucy', 'havana'), ('trusty', 'icehouse'), ('utopic', 'juno'), + ('vivid', 'kilo'), ]) @@ -64,6 +65,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2013.2', 'havana'), ('2014.1', 'icehouse'), ('2014.2', 'juno'), + ('2015.1', 'kilo'), ]) # The ugly duckling @@ -84,6 +86,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.0.0', 'juno'), ('2.1.0', 'juno'), ('2.2.0', 'juno'), + ('2.2.1', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' @@ -289,6 +292,9 @@ def configure_installation_source(rel): 'juno': 'trusty-updates/juno', 'juno/updates': 'trusty-updates/juno', 'juno/proposed': 'trusty-proposed/juno', + 'kilo': 'trusty-updates/kilo', + 'kilo/updates': 'trusty-updates/kilo', + 'kilo/proposed': 'trusty-proposed/kilo', } try: diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index d47dc228..1479f4f3 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -372,3 +372,46 @@ def ceph_version(): return None else: return None + + +class CephBrokerRq(object): + """Ceph broker request. + + Multiple operations can be added to a request and sent to the Ceph broker + to be executed. + + Request is json-encoded for sending over the wire. + + The API is versioned and defaults to version 1. + """ + def __init__(self, api_version=1): + self.api_version = api_version + self.ops = [] + + def add_op_create_pool(self, name, replica_count=3): + self.ops.append({'op': 'create-pool', 'name': name, + 'replicas': replica_count}) + + @property + def request(self): + return json.dumps({'api-version': self.api_version, 'ops': self.ops}) + + +class CephBrokerRsp(object): + """Ceph broker response. + + Response is json-decoded and contents provided as methods/properties. + + The API is versioned and defaults to version 1. + """ + def __init__(self, encoded_rsp): + self.api_version = None + self.rsp = json.loads(encoded_rsp) + + @property + def exit_code(self): + return self.rsp.get('exit-code') + + @property + def exit_msg(self): + return self.rsp.get('stderr') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c6f1680a..5221120c 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -162,13 +162,16 @@ def mkdir(path, owner='root', group='root', perms=0o555, force=False): uid = pwd.getpwnam(owner).pw_uid gid = grp.getgrnam(group).gr_gid realpath = os.path.abspath(path) - if os.path.exists(realpath): - if force and not os.path.isdir(realpath): + path_exists = os.path.exists(realpath) + if path_exists and force: + if not os.path.isdir(realpath): log("Removing non-directory file {} prior to mkdir()".format(path)) os.unlink(realpath) - else: + os.makedirs(realpath, perms) + os.chown(realpath, uid, gid) + elif not path_exists: os.makedirs(realpath, perms) - os.chown(realpath, uid, gid) + os.chown(realpath, uid, gid) def write_file(path, content, owner='root', group='root', perms=0o444): diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 0a126fc3..aceadea4 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -64,9 +64,16 @@ CLOUD_ARCHIVE_POCKETS = { 'trusty-juno/updates': 'trusty-updates/juno', 'trusty-updates/juno': 'trusty-updates/juno', 'juno/proposed': 'trusty-proposed/juno', - 'juno/proposed': 'trusty-proposed/juno', 'trusty-juno/proposed': 'trusty-proposed/juno', 'trusty-proposed/juno': 'trusty-proposed/juno', + # Kilo + 'kilo': 'trusty-updates/kilo', + 'trusty-kilo': 'trusty-updates/kilo', + 'trusty-kilo/updates': 'trusty-updates/kilo', + 'trusty-updates/kilo': 'trusty-updates/kilo', + 'kilo/proposed': 'trusty-proposed/kilo', + 'trusty-kilo/proposed': 'trusty-proposed/kilo', + 'trusty-proposed/kilo': 'trusty-proposed/kilo', } # The order of this list is very important. Handlers should be listed in from From 1643ccba98b9708206d360500f97edfa0887efeb Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 16 Jan 2015 13:31:26 +0000 Subject: [PATCH 14/61] Add missing files --- hooks/charmhelpers/core/decorators.py | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 hooks/charmhelpers/core/decorators.py diff --git a/hooks/charmhelpers/core/decorators.py b/hooks/charmhelpers/core/decorators.py new file mode 100644 index 00000000..029a4ef4 --- /dev/null +++ b/hooks/charmhelpers/core/decorators.py @@ -0,0 +1,41 @@ +# +# Copyright 2014 Canonical Ltd. +# +# Authors: +# Edward Hope-Morley +# + +import time + +from charmhelpers.core.hookenv import ( + log, + INFO, +) + + +def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): + """If the decorated function raises exception exc_type, allow num_retries + retry attempts before raise the exception. + """ + def _retry_on_exception_inner_1(f): + def _retry_on_exception_inner_2(*args, **kwargs): + retries = num_retries + multiplier = 1 + while True: + try: + return f(*args, **kwargs) + except exc_type: + if not retries: + raise + + delay = base_delay * multiplier + multiplier += 1 + log("Retrying '%s' %d more times (delay=%s)" % + (f.__name__, retries, delay), level=INFO) + retries -= 1 + if delay: + time.sleep(delay) + + return _retry_on_exception_inner_2 + + return _retry_on_exception_inner_1 From 3ffc9a84c89a7cd813c97797dbc474373dae7e1b Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 19 Jan 2015 09:23:38 +0000 Subject: [PATCH 15/61] Use centralized zeromq template --- charm-helpers-sync.yaml | 2 +- .../charmhelpers/contrib/openstack/context.py | 34 +++++++++++-------- .../contrib/openstack/templates/haproxy.cfg | 4 ++- .../contrib/openstack/templates}/zeromq | 8 ++++- .../contrib/storage/linux/ceph.py | 11 ++++++ templates/icehouse/neutron.conf | 3 +- 6 files changed, 43 insertions(+), 19 deletions(-) rename {templates/parts => hooks/charmhelpers/contrib/openstack/templates}/zeromq (53%) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 9b5e79e9..1a5cf078 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~openstack-charmers/charm-helpers/0mq destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 8ab61bf0..f63f0078 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -468,21 +468,25 @@ class HAProxyContext(OSContextGenerator): _unit = unit.replace('/', '-') cluster_hosts[laddr]['backends'][_unit] = _laddr - # NOTE(jamespage) no split configurations found, just use - # private addresses - if not cluster_hosts: - netmask = get_netmask_for_address(addr) - cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask), - 'backends': {l_unit: addr}} - for rid in relation_ids('cluster'): - for unit in related_units(rid): - _laddr = relation_get('private-address', - rid=rid, unit=unit) - if _laddr: - _unit = unit.replace('/', '-') - cluster_hosts[addr]['backends'][_unit] = _laddr + # NOTE(jamespage) add backend based on private address - this + # with either be the only backend or the fallback if no acls + # match in the frontend + cluster_hosts[addr] = {} + netmask = get_netmask_for_address(addr) + cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask), + 'backends': {l_unit: addr}} + for rid in relation_ids('cluster'): + for unit in related_units(rid): + _laddr = relation_get('private-address', + rid=rid, unit=unit) + if _laddr: + _unit = unit.replace('/', '-') + cluster_hosts[addr]['backends'][_unit] = _laddr - ctxt = {'frontends': cluster_hosts} + ctxt = { + 'frontends': cluster_hosts, + 'default_backend': addr + } if config('haproxy-server-timeout'): ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout') @@ -1001,6 +1005,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index 9ae1efb9..ad875f16 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -44,7 +44,9 @@ frontend tcp-in_{{ service }} {% for frontend in frontends -%} acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} - {% endfor %} + {% endfor -%} + default_backend {{ service }}_{{ default_backend }} + {% for frontend in frontends -%} backend {{ service }}_{{ frontend }} balance leastconn diff --git a/templates/parts/zeromq b/hooks/charmhelpers/contrib/openstack/templates/zeromq similarity index 53% rename from templates/parts/zeromq rename to hooks/charmhelpers/contrib/openstack/templates/zeromq index 3e32288c..ab796d93 100644 --- a/templates/parts/zeromq +++ b/hooks/charmhelpers/contrib/openstack/templates/zeromq @@ -1,6 +1,12 @@ {% if zmq_host -%} # ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) rpc_backend = zmq -rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing rpc_zmq_host = {{ zmq_host }} +{% if zmq_redis_address -%} +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis +[matchmaker_redis] +host = {{ zmq_redis_address }} +{% else -%} +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +{% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 1479f4f3..6ebeab5c 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -157,6 +157,17 @@ def create_keyring(service, key): log('Created new ceph keyring at %s.' % keyring, level=DEBUG) +def delete_keyring(service): + """Delete an existing Ceph keyring.""" + keyring = _keyring_path(service) + if not os.path.exists(keyring): + log('Keyring does not exist at %s' % keyring, level=WARNING) + return + + os.remove(keyring) + log('Deleted ring at %s.' % keyring, level=INFO) + + def create_key_file(service, key): """Create a file containing key.""" keyfile = _keyfile_path(service) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index c82ab09d..330ef617 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -27,8 +27,6 @@ notification_topics = notifications {% include "parts/rabbitmq" %} -{% include "parts/zeromq" %} - [QUOTAS] [DEFAULT_SERVICETYPE] @@ -39,3 +37,4 @@ root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf [keystone_authtoken] signing_dir = /var/lib/neutron/keystone-signing +{% include "zeromq" %} From d4ff5076c9e98a80a71ba2382666bacbe1a374e1 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 19 Jan 2015 10:27:38 +0000 Subject: [PATCH 16/61] Fixup zeromq sections --- templates/icehouse/neutron.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 330ef617..05fd10f9 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -27,6 +27,8 @@ notification_topics = notifications {% include "parts/rabbitmq" %} +{% include "zeromq" %} + [QUOTAS] [DEFAULT_SERVICETYPE] @@ -36,5 +38,3 @@ root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf [keystone_authtoken] signing_dir = /var/lib/neutron/keystone-signing - -{% include "zeromq" %} From 2330f3a11d49d2f51ba1d97feb724600a0087117 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 20 Jan 2015 11:37:44 +0000 Subject: [PATCH 17/61] Resync helper --- hooks/charmhelpers/contrib/openstack/templates/zeromq | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/zeromq index ab796d93..41d64666 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/zeromq +++ b/hooks/charmhelpers/contrib/openstack/templates/zeromq @@ -6,6 +6,8 @@ rpc_zmq_host = {{ zmq_host }} rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis [matchmaker_redis] host = {{ zmq_redis_address }} +matchmaker_heartbeat_freq = 15 +matchmaker_heartbeat_ttl = 30 {% else -%} rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing {% endif -%} From ba1f09fdfe4c7a512c96e13a53288fd7921133a3 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 20 Jan 2015 12:05:28 +0000 Subject: [PATCH 18/61] Resync helper --- hooks/charmhelpers/contrib/openstack/templates/zeromq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/zeromq index 41d64666..0695eef1 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/zeromq +++ b/hooks/charmhelpers/contrib/openstack/templates/zeromq @@ -4,10 +4,10 @@ rpc_backend = zmq rpc_zmq_host = {{ zmq_host }} {% if zmq_redis_address -%} rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis -[matchmaker_redis] -host = {{ zmq_redis_address }} matchmaker_heartbeat_freq = 15 matchmaker_heartbeat_ttl = 30 +[matchmaker_redis] +host = {{ zmq_redis_address }} {% else -%} rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing {% endif -%} From 3c7dd72d38828840c466f0f2c3093e1719e5aeda Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Feb 2015 13:31:39 +0000 Subject: [PATCH 19/61] Add dvr support --- hooks/neutron_ovs_context.py | 32 +++++++++++++++++++++----- hooks/neutron_ovs_hooks.py | 3 +++ hooks/neutron_ovs_utils.py | 23 ++++++++++++++++++- templates/juno/fwaas_driver.ini | 7 ++++++ templates/juno/l3_agent.ini | 7 ++++++ templates/juno/ml2_conf.ini | 40 +++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 templates/juno/fwaas_driver.ini create mode 100644 templates/juno/l3_agent.ini create mode 100644 templates/juno/ml2_conf.ini diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 7dbf5211..197c4420 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -1,3 +1,4 @@ +import ast from charmhelpers.core.hookenv import ( relation_ids, related_units, @@ -11,6 +12,7 @@ from charmhelpers.core.host import service_running, service_start from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.contrib.network.ip import get_address_in_network +from charmhelpers.contrib.openstack.context import OSContextGenerator import re @@ -26,17 +28,19 @@ def _neutron_api_settings(): 'neutron_security_groups': False, 'l2_population': True, 'overlay_network_type': 'gre', + 'enable_dvr': False, } for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) if 'l2-population' not in rdata: continue - neutron_settings = { - 'l2_population': rdata['l2-population'], - 'neutron_security_groups': rdata['neutron-security-groups'], - 'overlay_network_type': rdata['overlay-network-type'], - } + neutron_settings['l2_population'] = rdata['l2-population'] + if 'overlay-network-type' in rdata: + neutron_settings['overlay_network_type'] = \ + rdata['overlay-network-type'] + if 'enable-dvr' in rdata: + neutron_settings['enable_dvr'] = rdata['enable-dvr'] # Override with configuration if set to true if config('disable-security-groups'): neutron_settings['neutron_security_groups'] = False @@ -44,6 +48,11 @@ def _neutron_api_settings(): return neutron_settings +def use_dvr(): + api_settings = _neutron_api_settings() + return ast.literal_eval(api_settings['enable_dvr']) + + class OVSPluginContext(context.NeutronContext): interfaces = [] @@ -103,6 +112,7 @@ class OVSPluginContext(context.NeutronContext): neutron_api_settings = _neutron_api_settings() ovs_ctxt['neutron_security_groups'] = self.neutron_security_groups ovs_ctxt['l2_population'] = neutron_api_settings['l2_population'] + ovs_ctxt['distributed_routing'] = use_dvr() ovs_ctxt['overlay_network_type'] = \ neutron_api_settings['overlay_network_type'] # TODO: We need to sort out the syslog and debug/verbose options as a @@ -111,3 +121,15 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['verbose'] = conf['verbose'] ovs_ctxt['debug'] = conf['debug'] return ovs_ctxt + + +class L3AgentContext(OSContextGenerator): + + def __call__(self): + neutron_api_settings = _neutron_api_settings() + ctxt = {} + if neutron_api_settings['enable_dvr'] == 'True': + ctxt['agent_mode'] = 'dvr' + else: + ctxt['agent_mode'] = 'legacy' + return ctxt diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index eb53094d..ee314606 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -20,6 +20,7 @@ from charmhelpers.fetch import ( from neutron_ovs_utils import ( determine_packages, + determine_dvr_packages, register_configs, restart_map, ) @@ -41,6 +42,8 @@ def install(): @hooks.hook('config-changed') @restart_on_change(restart_map()) def config_changed(): + if determine_dvr_packages(): + apt_install(determine_dvr_packages(), fatal=True) CONFIGS.write_all() diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index b5d742de..c0381911 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -12,6 +12,8 @@ NOVA_CONF_DIR = "/etc/nova" NEUTRON_CONF_DIR = "/etc/neutron" NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR NEUTRON_DEFAULT = '/etc/default/neutron-server' +NEUTRON_L3_AGENT_CONF = "/etc/neutron/l3_agent.ini" +NEUTRON_FWAAS_CONF = "/etc/neutron/fwaas_driver.ini" ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR BASE_RESOURCE_MAP = OrderedDict([ @@ -24,12 +26,29 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-plugin-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext()], }), + (NEUTRON_L3_AGENT_CONF, { + 'services': ['neutron-vpn-agent'], + 'contexts': [neutron_ovs_context.L3AgentContext()], + }), + (NEUTRON_FWAAS_CONF, { + 'services': ['neutron-vpn-agent'], + 'contexts': [neutron_ovs_context.L3AgentContext()], + }), ]) TEMPLATES = 'templates/' +def determine_dvr_packages(): + pkgs = [] + if neutron_ovs_context.use_dvr(): + pkgs = 'neutron-vpn-agent' + return pkgs + + def determine_packages(): - return neutron_plugin_attribute('ovs', 'packages', 'neutron') + pkgs = neutron_plugin_attribute('ovs', 'packages', 'neutron') + pkgs.extend(determine_dvr_packages()) + return pkgs def register_configs(release=None): @@ -47,6 +66,8 @@ def resource_map(): hook execution. ''' resource_map = deepcopy(BASE_RESOURCE_MAP) + if not neutron_ovs_context.use_dvr(): + resource_map.pop(NEUTRON_L3_AGENT_CONF) return resource_map diff --git a/templates/juno/fwaas_driver.ini b/templates/juno/fwaas_driver.ini new file mode 100644 index 00000000..e64046dc --- /dev/null +++ b/templates/juno/fwaas_driver.ini @@ -0,0 +1,7 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[fwaas] +driver = neutron.services.firewall.drivers.linux.iptables_fwaas.IptablesFwaasDriver +enabled = True diff --git a/templates/juno/l3_agent.ini b/templates/juno/l3_agent.ini new file mode 100644 index 00000000..8e93c71a --- /dev/null +++ b/templates/juno/l3_agent.ini @@ -0,0 +1,7 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +[DEFAULT] +interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver +agent_mode = {{ agent_mode }} diff --git a/templates/juno/ml2_conf.ini b/templates/juno/ml2_conf.ini new file mode 100644 index 00000000..1a0c7c93 --- /dev/null +++ b/templates/juno/ml2_conf.ini @@ -0,0 +1,40 @@ +# icehouse +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +# Config managed by neutron-openvswitch charm +############################################################################### +[ml2] +type_drivers = gre,vxlan,vlan,flat +tenant_network_types = gre,vxlan,vlan,flat +mechanism_drivers = openvswitch,hyperv,l2population + +[ml2_type_gre] +tunnel_id_ranges = 1:1000 + +[ml2_type_vxlan] +vni_ranges = 1001:2000 + +[ml2_type_vlan] +network_vlan_ranges = physnet1:1000:2000 + +[ml2_type_flat] +flat_networks = physnet1 + +[ovs] +enable_tunneling = True +local_ip = {{ local_ip }} +bridge_mappings = physnet1:br-data + +[agent] +tunnel_types = {{ overlay_network_type }} +l2_population = {{ l2_population }} +enable_distributed_routing = {{ distributed_routing }} + +[securitygroup] +{% if neutron_security_groups -%} +enable_security_group = True +firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver +{% else -%} +enable_security_group = False +{% endif -%} From 5ab3daeb05e796558aa411e89a9f1b38ba5856e7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Feb 2015 15:12:56 +0000 Subject: [PATCH 20/61] Fix defaults as templates expect strings not booleans --- hooks/neutron_ovs_context.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 197c4420..f6305a7c 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -20,15 +20,21 @@ OVS_BRIDGE = 'br-int' DATA_BRIDGE = 'br-data' +def to_boolean(option): + if option is None: + return False + return ast.literal_eval(option) + + def _neutron_api_settings(): ''' Inspects current neutron-plugin relation ''' neutron_settings = { - 'neutron_security_groups': False, - 'l2_population': True, + 'neutron_security_groups': 'False', + 'l2_population': 'True', 'overlay_network_type': 'gre', - 'enable_dvr': False, + 'enable_dvr': 'False', } for rid in relation_ids('neutron-plugin-api'): for unit in related_units(rid): @@ -50,7 +56,7 @@ def _neutron_api_settings(): def use_dvr(): api_settings = _neutron_api_settings() - return ast.literal_eval(api_settings['enable_dvr']) + return to_boolean(api_settings['enable_dvr']) class OVSPluginContext(context.NeutronContext): From 2f1aceb741d4517bd7f0ecf5567a0c538c2371bd Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Feb 2015 15:26:32 +0000 Subject: [PATCH 21/61] Add setup for configuring an external bridge --- config.yaml | 9 +++++++ hooks/neutron_ovs_context.py | 49 +++++++++++++++++++++++++++++++++++- hooks/neutron_ovs_utils.py | 22 ++++++++++++++++ templates/ext-port.conf | 9 +++++++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 templates/ext-port.conf diff --git a/config.yaml b/config.yaml index ea3ed841..c424956c 100644 --- a/config.yaml +++ b/config.yaml @@ -47,3 +47,12 @@ options: . This network will be used for tenant network traffic in overlay networks. + ext-port: + type: string + default: + description: | + A space-separated list of external ports to use for routing of instance + traffic to the external public network. Valid values are either MAC + addresses (in which case only MAC addresses for interfaces without an IP + address already assigned will be used), or interfaces (eth0) + diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index f6305a7c..06a6bbec 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -6,12 +6,17 @@ from charmhelpers.core.hookenv import ( config, unit_get, ) +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + get_ipv4_addr, + get_ipv6_addr, + is_bridge_member, +) from charmhelpers.core.host import list_nics, get_nic_hwaddr from charmhelpers.contrib.openstack import context from charmhelpers.core.host import service_running, service_start from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip -from charmhelpers.contrib.network.ip import get_address_in_network from charmhelpers.contrib.openstack.context import OSContextGenerator import re @@ -139,3 +144,45 @@ class L3AgentContext(OSContextGenerator): else: ctxt['agent_mode'] = 'legacy' return ctxt + + +class NeutronPortContext(OSContextGenerator): + + def _resolve_port(self, config_key): + if not config(config_key): + return None + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(['eth', 'bond']): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + \ + get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in config(config_key).split(): + entry = entry.strip() + if re.match(mac_regex, entry): + if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + return hwaddr_to_nic[entry] + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + return entry + return None + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + port = self._resolve_port('ext-port') + if port: + return {"ext_port": port} + else: + return None diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index c0381911..b3377cc2 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -7,6 +7,10 @@ from charmhelpers.contrib.openstack.utils import ( os_release, ) import neutron_ovs_context +from charmhelpers.contrib.network.ovs import ( + add_bridge, + add_bridge_port, +) NOVA_CONF_DIR = "/etc/nova" NEUTRON_CONF_DIR = "/etc/neutron" @@ -15,6 +19,7 @@ NEUTRON_DEFAULT = '/etc/default/neutron-server' NEUTRON_L3_AGENT_CONF = "/etc/neutron/l3_agent.ini" NEUTRON_FWAAS_CONF = "/etc/neutron/fwaas_driver.ini" ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR +EXT_PORT_CONF = '/etc/init/ext-port.conf' BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { @@ -34,8 +39,15 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-vpn-agent'], 'contexts': [neutron_ovs_context.L3AgentContext()], }), + (EXT_PORT_CONF, { + 'services': [], + 'contexts': [neutron_ovs_context.ExternalPortContext()], + }), ]) TEMPLATES = 'templates/' +INT_BRIDGE = "br-int" +EXT_BRIDGE = "br-ex" +DATA_BRIDGE = 'br-data' def determine_dvr_packages(): @@ -77,3 +89,13 @@ def restart_map(): state. ''' return {k: v['services'] for k, v in resource_map().iteritems()} + + +def configure_ovs(): + add_bridge(INT_BRIDGE) + add_bridge(EXT_BRIDGE) + ext_port_ctx = neutron_ovs_context.ExternalPortContext()() + if ext_port_ctx and ext_port_ctx['ext_port']: + add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) + + add_bridge(DATA_BRIDGE) diff --git a/templates/ext-port.conf b/templates/ext-port.conf new file mode 100644 index 00000000..6080c30e --- /dev/null +++ b/templates/ext-port.conf @@ -0,0 +1,9 @@ +description "Enabling Quantum external networking port" + +start on runlevel [2345] + +task + +script + ip link set {{ ext_port }} up +end script \ No newline at end of file From d89c19dd78ba247f67a2c2b0ae45db93ae8b6fbf Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Feb 2015 15:28:26 +0000 Subject: [PATCH 22/61] Call ovs setup --- hooks/neutron_ovs_hooks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index ee314606..848229bf 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -19,6 +19,7 @@ from charmhelpers.fetch import ( ) from neutron_ovs_utils import ( + configure_ovs, determine_packages, determine_dvr_packages, register_configs, @@ -44,6 +45,7 @@ def install(): def config_changed(): if determine_dvr_packages(): apt_install(determine_dvr_packages(), fatal=True) + configure_ovs() CONFIGS.write_all() From 35afca213d3ff5979f3d66150e9053751e8cb5da Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Feb 2015 13:35:41 +0000 Subject: [PATCH 23/61] Tmp disable configure_ovs --- hooks/neutron_ovs_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 848229bf..a410cc4c 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -45,7 +45,7 @@ def install(): def config_changed(): if determine_dvr_packages(): apt_install(determine_dvr_packages(), fatal=True) - configure_ovs() + #configure_ovs() CONFIGS.write_all() From a0c85efdd404bf02c43af37daed102eb3bf7c3b8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Feb 2015 14:17:30 +0000 Subject: [PATCH 24/61] Reenable configure_ovs --- hooks/neutron_ovs_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index a410cc4c..848229bf 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -45,7 +45,7 @@ def install(): def config_changed(): if determine_dvr_packages(): apt_install(determine_dvr_packages(), fatal=True) - #configure_ovs() + configure_ovs() CONFIGS.write_all() From c02f501399abccf51164c8910a95c61477c5030e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 4 Feb 2015 16:30:03 +0000 Subject: [PATCH 25/61] Add relation to nova-cc (as neutron-gateway has) to get ks service info --- .../neutron-network-service-relation-changed | 1 + hooks/neutron-network-service-relation-joined | 1 + hooks/neutron_ovs_context.py | 54 ++++++++++++++++++- hooks/neutron_ovs_hooks.py | 10 ++++ hooks/neutron_ovs_utils.py | 19 +++++-- .../quantum-network-service-relation-changed | 1 + metadata.yaml | 2 + templates/juno/metadata_agent.ini | 19 +++++++ 8 files changed, 103 insertions(+), 4 deletions(-) create mode 120000 hooks/neutron-network-service-relation-changed create mode 120000 hooks/neutron-network-service-relation-joined create mode 120000 hooks/quantum-network-service-relation-changed create mode 100644 templates/juno/metadata_agent.ini diff --git a/hooks/neutron-network-service-relation-changed b/hooks/neutron-network-service-relation-changed new file mode 120000 index 00000000..55aa8e52 --- /dev/null +++ b/hooks/neutron-network-service-relation-changed @@ -0,0 +1 @@ +neutron_ovs_hooks.py \ No newline at end of file diff --git a/hooks/neutron-network-service-relation-joined b/hooks/neutron-network-service-relation-joined new file mode 120000 index 00000000..55aa8e52 --- /dev/null +++ b/hooks/neutron-network-service-relation-joined @@ -0,0 +1 @@ +neutron_ovs_hooks.py \ No newline at end of file diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 06a6bbec..3ff887f3 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -1,4 +1,6 @@ import ast +import os +import uuid from charmhelpers.core.hookenv import ( relation_ids, related_units, @@ -12,12 +14,16 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr, is_bridge_member, ) +from charmhelpers.contrib.openstack.ip import resolve_address from charmhelpers.core.host import list_nics, get_nic_hwaddr from charmhelpers.contrib.openstack import context from charmhelpers.core.host import service_running, service_start from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip -from charmhelpers.contrib.openstack.context import OSContextGenerator +from charmhelpers.contrib.openstack.context import ( + OSContextGenerator, + context_complete, +) import re @@ -186,3 +192,49 @@ class ExternalPortContext(NeutronPortContext): return {"ext_port": port} else: return None + + +class NetworkServiceContext(OSContextGenerator): + interfaces = ['neutron-network-service'] + + def __call__(self): + for rid in relation_ids('neutron-network-service'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'region': rdata.get('region'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + } + if context_complete(ctxt): + return ctxt + + +class DVRSharedSecretContext(OSContextGenerator): + + def get_shared_secret(self): + secret = None + if not os.path.exists(self.SHARED_SECRET): + secret = str(uuid.uuid4()) + with open(self.SHARED_SECRET, 'w') as secret_file: + secret_file.write(secret) + else: + with open(self.SHARED_SECRET, 'r') as secret_file: + secret = secret_file.read().strip() + return secret + + def __call__(self): + self.SHARED_SECRET = "/etc/neutron/secret.txt" + if use_dvr(): + ctxt = { + 'shared_secret': self.get_shared_secret(), + 'local_ip': resolve_address(), + } + else: + ctxt = {} + return ctxt diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 848229bf..348ab115 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -22,6 +22,7 @@ from neutron_ovs_utils import ( configure_ovs, determine_packages, determine_dvr_packages, + get_shared_secret, register_configs, restart_map, ) @@ -38,6 +39,7 @@ def install(): apt_install(pkg, fatal=True) +@hooks.hook('neutron-network-service-relation-changed') @hooks.hook('neutron-plugin-relation-changed') @hooks.hook('neutron-plugin-api-relation-changed') @hooks.hook('config-changed') @@ -49,6 +51,14 @@ def config_changed(): CONFIGS.write_all() +@hooks.hook('neutron-plugin-relation-joined') +def neutron_plugin_joined(relation_id=None): + rel_data = { + 'metadata-shared-secret': get_shared_secret() + } + relation_set(relation_id=relation_id, **rel_data) + + @hooks.hook('amqp-relation-joined') def amqp_joined(relation_id=None): relation_set(relation_id=relation_id, diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index b3377cc2..5c5040cf 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -20,15 +20,17 @@ NEUTRON_L3_AGENT_CONF = "/etc/neutron/l3_agent.ini" NEUTRON_FWAAS_CONF = "/etc/neutron/fwaas_driver.ini" ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR EXT_PORT_CONF = '/etc/init/ext-port.conf' +NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini" + BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { - 'services': ['neutron-plugin-openvswitch-agent'], + 'services': ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent', 'neutron-openvswitch-agent', 'neutron-vpn-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext(), context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR)], }), (ML2_CONF, { - 'services': ['neutron-plugin-openvswitch-agent'], + 'services': ['neutron-plugin-openvswitch-agent', 'neutron-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext()], }), (NEUTRON_L3_AGENT_CONF, { @@ -43,6 +45,11 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': [], 'contexts': [neutron_ovs_context.ExternalPortContext()], }), + (NEUTRON_METADATA_AGENT_CONF, { + 'services': ['neutron-metadata-agent'], + 'contexts': [neutron_ovs_context.DVRSharedSecretContext(), + neutron_ovs_context.NetworkServiceContext()], + }), ]) TEMPLATES = 'templates/' INT_BRIDGE = "br-int" @@ -53,7 +60,7 @@ DATA_BRIDGE = 'br-data' def determine_dvr_packages(): pkgs = [] if neutron_ovs_context.use_dvr(): - pkgs = 'neutron-vpn-agent' + pkgs = ['neutron-vpn-agent'] return pkgs @@ -99,3 +106,9 @@ def configure_ovs(): add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) add_bridge(DATA_BRIDGE) + + +def get_shared_secret(): + ctxt = neutron_ovs_context.DVRSharedSecretContext()() + if 'shared_secret' in ctxt: + return ctxt['shared_secret'] diff --git a/hooks/quantum-network-service-relation-changed b/hooks/quantum-network-service-relation-changed new file mode 120000 index 00000000..55aa8e52 --- /dev/null +++ b/hooks/quantum-network-service-relation-changed @@ -0,0 +1 @@ +neutron_ovs_hooks.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 0e840258..790cda80 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -20,6 +20,8 @@ provides: neutron-plugin: interface: neutron-plugin scope: container + neutron-network-service: + interface: quantum requires: amqp: interface: rabbitmq diff --git a/templates/juno/metadata_agent.ini b/templates/juno/metadata_agent.ini new file mode 100644 index 00000000..3062d697 --- /dev/null +++ b/templates/juno/metadata_agent.ini @@ -0,0 +1,19 @@ +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +############################################################################### +# Metadata service seems to cache neutron api url from keystone so trigger +# restart if it changes: {{ quantum_url }} +[DEFAULT] +auth_url = {{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 +auth_region = {{ region }} +admin_tenant_name = {{ service_tenant }} +admin_user = {{ service_username }} +admin_password = {{ service_password }} +root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf +state_path = /var/lib/neutron +# Gateway runs a metadata API server locally +#nova_metadata_ip = {{ local_ip }} +nova_metadata_port = 8775 +metadata_proxy_shared_secret = {{ shared_secret }} +cache_url = memory://?default_ttl=5 From b9fd91de35573b8e1a2cc26201ef42438cb37298 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 5 Feb 2015 09:57:47 +0000 Subject: [PATCH 26/61] Fix BASE_RESOURCE_MAP --- hooks/neutron_ovs_hooks.py | 1 + hooks/neutron_ovs_utils.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 348ab115..88b97189 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -46,6 +46,7 @@ def install(): @restart_on_change(restart_map()) def config_changed(): if determine_dvr_packages(): + apt_update() apt_install(determine_dvr_packages(), fatal=True) configure_ovs() CONFIGS.write_all() diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 5c5040cf..b5314b5d 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -25,12 +25,12 @@ NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini" BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { - 'services': ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent', 'neutron-openvswitch-agent', 'neutron-vpn-agent'], + 'services': ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent', 'neutron-vpn-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext(), context.AMQPContext(ssl_dir=NEUTRON_CONF_DIR)], }), (ML2_CONF, { - 'services': ['neutron-plugin-openvswitch-agent', 'neutron-openvswitch-agent'], + 'services': ['neutron-plugin-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext()], }), (NEUTRON_L3_AGENT_CONF, { @@ -42,7 +42,7 @@ BASE_RESOURCE_MAP = OrderedDict([ 'contexts': [neutron_ovs_context.L3AgentContext()], }), (EXT_PORT_CONF, { - 'services': [], + 'services': ['neutron-vpn-agent'], 'contexts': [neutron_ovs_context.ExternalPortContext()], }), (NEUTRON_METADATA_AGENT_CONF, { From 63e451fbcfd4b10f70784578cd741f6635835796 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 20 Feb 2015 08:17:24 +0000 Subject: [PATCH 27/61] Inform nova-compute if dvr is enabled as it'll need to enable the metadata service --- hooks/neutron_ovs_hooks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 88b97189..8ce3d5d0 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -8,6 +8,7 @@ from charmhelpers.core.hookenv import ( config, log, relation_set, + relation_ids, ) from charmhelpers.core.host import ( @@ -41,7 +42,6 @@ def install(): @hooks.hook('neutron-network-service-relation-changed') @hooks.hook('neutron-plugin-relation-changed') -@hooks.hook('neutron-plugin-api-relation-changed') @hooks.hook('config-changed') @restart_on_change(restart_map()) def config_changed(): @@ -51,6 +51,17 @@ def config_changed(): configure_ovs() CONFIGS.write_all() +@hooks.hook('neutron-plugin-api-relation-changed') +@restart_on_change(restart_map()) +def neutron_plugin_api_changed(): + if determine_dvr_packages(): + apt_update() + apt_install(determine_dvr_packages(), fatal=True) + configure_ovs() + CONFIGS.write_all() + # If dvr setting has changed, need to pass that on + for rid in relation_ids('neutron-plugin'): + neutron_plugin_joined(relation_id=rid) @hooks.hook('neutron-plugin-relation-joined') def neutron_plugin_joined(relation_id=None): From 02dca40b5e98a34f007da840c2500bd301bcf29a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 20 Feb 2015 09:13:08 +0000 Subject: [PATCH 28/61] Fix bug stopping agent type being set correctly. neutron_api_settings['enable_dvr'] is now a boolean not a string --- hooks/neutron_ovs_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index f97991d2..60ed1e15 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -147,7 +147,7 @@ class L3AgentContext(OSContextGenerator): def __call__(self): neutron_api_settings = _neutron_api_settings() ctxt = {} - if neutron_api_settings['enable_dvr'] == 'True': + if neutron_api_settings['enable_dvr']: ctxt['agent_mode'] = 'dvr' else: ctxt['agent_mode'] = 'legacy' From a8e25ad926cc4d6a2412b79dce4af1aae4600367 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Feb 2015 11:46:08 +0000 Subject: [PATCH 29/61] Switch back to trunk charm-helpers --- charm-helpers-sync.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 1a5cf078..9b5e79e9 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:~openstack-charmers/charm-helpers/0mq +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core From d1917d8b0c428191f07445e32c535d641a579394 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Feb 2015 11:48:55 +0000 Subject: [PATCH 30/61] Specialize configuration for juno + zeromq --- templates/icehouse/neutron.conf | 2 -- templates/juno/neutron.conf | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 templates/juno/neutron.conf diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 05fd10f9..93c7043e 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -27,8 +27,6 @@ notification_topics = notifications {% include "parts/rabbitmq" %} -{% include "zeromq" %} - [QUOTAS] [DEFAULT_SERVICETYPE] diff --git a/templates/juno/neutron.conf b/templates/juno/neutron.conf new file mode 100644 index 00000000..05fd10f9 --- /dev/null +++ b/templates/juno/neutron.conf @@ -0,0 +1,40 @@ +# grizzly +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +# Config managed by neutron-openvswitch charm +############################################################################### +[DEFAULT] +verbose = {{ verbose }} +debug = {{ debug }} +use_syslog = {{ use_syslog }} +state_path = /var/lib/neutron +lock_path = $state_path/lock +bind_host = 0.0.0.0 +bind_port = 9696 + +{% if core_plugin -%} +core_plugin = {{ core_plugin }} +{% endif -%} + +api_paste_config = /etc/neutron/api-paste.ini +auth_strategy = keystone +{% if notifications == 'True' -%} +notification_driver = neutron.openstack.common.notifier.rpc_notifier +{% endif -%} +default_notification_level = INFO +notification_topics = notifications + +{% include "parts/rabbitmq" %} + +{% include "zeromq" %} + +[QUOTAS] + +[DEFAULT_SERVICETYPE] + +[AGENT] +root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf + +[keystone_authtoken] +signing_dir = /var/lib/neutron/keystone-signing From 674d60e985fe43195127e58853fc606e739e6887 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Feb 2015 11:53:24 +0000 Subject: [PATCH 31/61] Update headers for templates --- templates/icehouse/neutron.conf | 2 +- templates/juno/neutron.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 93c7043e..0b1a8fb4 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -1,4 +1,4 @@ -# grizzly +# icehouse ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten. diff --git a/templates/juno/neutron.conf b/templates/juno/neutron.conf index 05fd10f9..b08aa12f 100644 --- a/templates/juno/neutron.conf +++ b/templates/juno/neutron.conf @@ -1,4 +1,4 @@ -# grizzly +# juno ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten. From aacb32c89ed59e30809f37e12efd346ce20569fc Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Feb 2015 13:53:48 +0000 Subject: [PATCH 32/61] Add kilo zeromq template --- templates/kilo/zeromq | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 templates/kilo/zeromq diff --git a/templates/kilo/zeromq b/templates/kilo/zeromq new file mode 100644 index 00000000..873be80e --- /dev/null +++ b/templates/kilo/zeromq @@ -0,0 +1,14 @@ +{% if zmq_host -%} +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) +rpc_backend = zmq +rpc_zmq_host = {{ zmq_host }} +{% if zmq_redis_address -%} +rpc_zmq_matchmaker = oslo_messaging._drivers.matchmaker_redis.MatchMakerRedis +matchmaker_heartbeat_freq = 15 +matchmaker_heartbeat_ttl = 30 +[matchmaker_redis] +host = {{ zmq_redis_address }} +{% else -%} +rpc_zmq_matchmaker = oslo_messaging._drivers.matchmaker_ring.MatchMakerRing +{% endif -%} +{% endif -%} From 030f9b9c280c80a1ce3d8b1ea8c768c6ff9dbc85 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Feb 2015 16:23:38 +0000 Subject: [PATCH 33/61] Added unit tests --- unit_tests/test_neutron_ovs_context.py | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 631b37c1..80bd363d 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -90,6 +90,7 @@ class OVSPluginContextTest(CharmTestCase): self.relation_ids.return_value = ['rid2'] self.test_relation.set({'neutron-security-groups': 'True', 'l2-population': 'True', + 'enable-dvr': 'True', 'overlay-network-type': 'gre', }) self.get_host_ip.return_value = '127.0.0.15' @@ -98,7 +99,7 @@ class OVSPluginContextTest(CharmTestCase): expect = { 'neutron_alchemy_flags': {}, 'neutron_security_groups': True, - 'distributed_routing': False, + 'distributed_routing': True, 'verbose': True, 'local_ip': '127.0.0.15', 'config': 'neutron.randomconfig', @@ -167,3 +168,31 @@ class OVSPluginContextTest(CharmTestCase): } self.assertEquals(expect, napi_ctxt()) self.service_start.assertCalled() + +class L3AgentContextTest(CharmTestCase): + + def setUp(self): + super(L3AgentContextTest, self).setUp(context, TO_PATCH) + self.relation_get.side_effect = self.test_relation.get + self.config.side_effect = self.test_config.get + + def tearDown(self): + super(L3AgentContextTest, self).tearDown() + + def test_dvr_enabled(self): + self.related_units.return_value = ['unit1'] + self.relation_ids.return_value = ['rid2'] + self.test_relation.set({'neutron-security-groups': 'True', + 'enable-dvr': 'True', + 'l2-population': 'True', + 'overlay-network-type': 'vxlan'}) + self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'dvr'}) + + def test_dvr_disabled(self): + self.related_units.return_value = ['unit1'] + self.relation_ids.return_value = ['rid2'] + self.test_relation.set({'neutron-security-groups': 'True', + 'enable-dvr': 'False', + 'l2-population': 'True', + 'overlay-network-type': 'vxlan'}) + self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'legacy'}) From f5321e3f5076388b18551b7844a6113b7716d58b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Feb 2015 07:54:14 +0000 Subject: [PATCH 34/61] Use charm-helpers ExternalPortContext --- charm-helpers-sync.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 6 +- .../charmhelpers/contrib/openstack/context.py | 96 +++++++++++++++++-- .../contrib/openstack/files/__init__.py | 18 ++++ .../contrib/openstack/files/check_haproxy.sh | 32 +++++++ .../files/check_haproxy_queue_depth.sh | 30 ++++++ hooks/charmhelpers/contrib/openstack/ip.py | 37 +++++++ .../contrib/openstack/templates/zeromq | 14 +++ hooks/charmhelpers/contrib/openstack/utils.py | 1 + hooks/charmhelpers/core/fstab.py | 4 +- hooks/neutron_ovs_context.py | 42 -------- hooks/neutron_ovs_utils.py | 2 +- 12 files changed, 228 insertions(+), 56 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/files/__init__.py create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh create mode 100755 hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh create mode 100644 hooks/charmhelpers/contrib/openstack/templates/zeromq diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 9b5e79e9..35c175f4 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~gnuoy/charm-helpers/neutron-contexts destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index 9a2588b6..9333efc3 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -48,6 +48,9 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.decorators import ( retry_on_exception, ) +from charmhelpers.core.strutils import ( + bool_from_string, +) class HAIncompleteConfig(Exception): @@ -164,7 +167,8 @@ def https(): . returns: boolean ''' - if config_get('use-https') == "yes": + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): return True if config_get('ssl_cert') and config_get('ssl_key'): return True diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c7c4cd4a..9385f1a2 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -19,9 +19,14 @@ import os import time from base64 import b64decode from subprocess import check_call +import re import six +from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr +) from charmhelpers.fetch import ( apt_install, filter_installed_packages, @@ -66,10 +71,12 @@ from charmhelpers.contrib.openstack.neutron import ( ) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_ipv4_addr, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, is_address_in_network, + is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip @@ -279,9 +286,25 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] + def __init__(self, service=None, service_user=None): + self.service = service + self.service_user = service_user + def __call__(self): log('Generating template context for identity-service', level=DEBUG) ctxt = {} + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -291,15 +314,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -1021,6 +1045,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt @@ -1052,3 +1078,55 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronPortContext(OSContextGenerator): + + def _resolve_port(self, config_key): + if not config(config_key): + return None + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(['eth', 'bond']): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + \ + get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in config(config_key).split(): + entry = entry.strip() + if re.match(mac_regex, entry): + if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + return hwaddr_to_nic[entry] + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + return entry + return None + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + port = self._resolve_port('ext-port') + if port: + return {"ext_port": port} + else: + return None + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + port = self._resolve_port('data-port') + if port: + return {"data_port": port} + else: + return None diff --git a/hooks/charmhelpers/contrib/openstack/files/__init__.py b/hooks/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 00000000..75876796 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh new file mode 100755 index 00000000..eb8527f5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +CRITICAL=0 +NOTACTIVE='' +LOGFILE=/var/log/nagios/check_haproxy.log +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); +do + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') + if [ $? != 0 ]; then + date >> $LOGFILE + echo $output >> $LOGFILE + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 + CRITICAL=1 + NOTACTIVE="${NOTACTIVE} $appserver" + fi +done + +if [ $CRITICAL = 1 ]; then + echo "CRITICAL:${NOTACTIVE}" + exit 2 +fi + +echo "OK: All haproxy instances looking good" +exit 0 diff --git a/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh new file mode 100755 index 00000000..3ebb5329 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +# These should be config options at some stage +CURRQthrsh=0 +MAXQthrsh=100 + +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') + +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) + +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') +do + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) + + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" + exit 2 + fi +done + +echo "OK: All haproxy queue depths looking good" +exit 0 + diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py index 9eabed73..29bbddcb 100644 --- a/hooks/charmhelpers/contrib/openstack/ip.py +++ b/hooks/charmhelpers/contrib/openstack/ip.py @@ -26,6 +26,8 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.hahelpers.cluster import is_clustered +from functools import partial + PUBLIC = 'public' INTERNAL = 'int' ADMIN = 'admin' @@ -107,3 +109,38 @@ def resolve_address(endpoint_type=PUBLIC): "clustered=%s)" % (net_type, clustered)) return resolved_address + + +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, + override=None): + """Returns the correct endpoint URL to advertise to Keystone. + + This method provides the correct endpoint URL which should be advertised to + the keystone charm for endpoint creation. This method allows for the url to + be overridden to force a keystone endpoint to have specific URL for any of + the defined scopes (admin, internal, public). + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param url_template: str format string for creating the url template. Only + two values will be passed - the scheme+hostname + returned by the canonical_url and the port. + :param endpoint_type: str endpoint type to resolve. + :param override: str the name of the config option which overrides the + endpoint URL defined by the charm itself. None will + disable any overrides (default). + """ + if override: + # Return any user-defined overrides for the keystone endpoint URL. + user_value = config(override) + if user_value: + return user_value.strip() + + return url_template % (canonical_url(configs, endpoint_type), port) + + +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) + +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) + +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) diff --git a/hooks/charmhelpers/contrib/openstack/templates/zeromq b/hooks/charmhelpers/contrib/openstack/templates/zeromq new file mode 100644 index 00000000..0695eef1 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/zeromq @@ -0,0 +1,14 @@ +{% if zmq_host -%} +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) +rpc_backend = zmq +rpc_zmq_host = {{ zmq_host }} +{% if zmq_redis_address -%} +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis +matchmaker_heartbeat_freq = 15 +matchmaker_heartbeat_ttl = 30 +[matchmaker_redis] +host = {{ zmq_redis_address }} +{% else -%} +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing +{% endif -%} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 26259a03..af2b3596 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,6 +103,7 @@ SWIFT_CODENAMES = OrderedDict([ ('2.1.0', 'juno'), ('2.2.0', 'juno'), ('2.2.1', 'kilo'), + ('2.2.2', 'kilo'), ]) DEFAULT_LOOPBACK_SIZE = '5G' diff --git a/hooks/charmhelpers/core/fstab.py b/hooks/charmhelpers/core/fstab.py index 9cdcc886..3056fbac 100644 --- a/hooks/charmhelpers/core/fstab.py +++ b/hooks/charmhelpers/core/fstab.py @@ -77,7 +77,7 @@ class Fstab(io.FileIO): for line in self.readlines(): line = line.decode('us-ascii') try: - if line.strip() and not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): yield self._hydrate_entry(line) except ValueError: pass @@ -104,7 +104,7 @@ class Fstab(io.FileIO): found = False for index, line in enumerate(lines): - if not line.startswith("#"): + if line.strip() and not line.strip().startswith("#"): if self._hydrate_entry(line) == entry: found = True break diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 60ed1e15..df4cfdd3 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -154,48 +154,6 @@ class L3AgentContext(OSContextGenerator): return ctxt -class NeutronPortContext(OSContextGenerator): - - def _resolve_port(self, config_key): - if not config(config_key): - return None - hwaddr_to_nic = {} - hwaddr_to_ip = {} - for nic in list_nics(['eth', 'bond']): - hwaddr = get_nic_hwaddr(nic) - hwaddr_to_nic[hwaddr] = nic - addresses = get_ipv4_addr(nic, fatal=False) + \ - get_ipv6_addr(iface=nic, fatal=False) - hwaddr_to_ip[hwaddr] = addresses - mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) - for entry in config(config_key).split(): - entry = entry.strip() - if re.match(mac_regex, entry): - if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: - # If the nic is part of a bridge then don't use it - if is_bridge_member(hwaddr_to_nic[entry]): - continue - # Entry is a MAC address for a valid interface that doesn't - # have an IP address assigned yet. - return hwaddr_to_nic[entry] - else: - # If the passed entry is not a MAC address, assume it's a valid - # interface, and that the user put it there on purpose (we can - # trust it to be the real external network). - return entry - return None - - -class ExternalPortContext(NeutronPortContext): - - def __call__(self): - port = self._resolve_port('ext-port') - if port: - return {"ext_port": port} - else: - return None - - class NetworkServiceContext(OSContextGenerator): interfaces = ['neutron-network-service'] diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index ec5168f9..1bac5f42 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -45,7 +45,7 @@ DVR_RESOURCE_MAP = OrderedDict([ }), (EXT_PORT_CONF, { 'services': ['neutron-vpn-agent'], - 'contexts': [neutron_ovs_context.ExternalPortContext()], + 'contexts': [context.ExternalPortContext()], }), (NEUTRON_METADATA_AGENT_CONF, { 'services': ['neutron-metadata-agent'], From 0990ac13b331c2c5e84e940f94816cfdb622387e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Feb 2015 08:15:08 +0000 Subject: [PATCH 35/61] Fix ExternalPortContext location --- hooks/neutron_ovs_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 1bac5f42..f64d58ff 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -105,7 +105,7 @@ def restart_map(): def configure_ovs(): add_bridge(INT_BRIDGE) add_bridge(EXT_BRIDGE) - ext_port_ctx = neutron_ovs_context.ExternalPortContext()() + ext_port_ctx = context.ExternalPortContext()() if ext_port_ctx and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) From 8b5d14e345f58f35bbce5f231b64df17d4cc6ccd Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Feb 2015 13:50:38 +0000 Subject: [PATCH 36/61] More unit tests --- hooks/neutron_ovs_context.py | 32 +++++----- hooks/neutron_ovs_hooks.py | 2 + unit_tests/test_neutron_ovs_context.py | 81 ++++++++++++++++++++++++++ unit_tests/test_neutron_ovs_hooks.py | 36 ++++++++++++ unit_tests/test_neutron_ovs_utils.py | 44 +++++++++++++- 5 files changed, 178 insertions(+), 17 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index df4cfdd3..62bec11a 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -9,9 +9,6 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.network.ip import ( get_address_in_network, - get_ipv4_addr, - get_ipv6_addr, - is_bridge_member, ) from charmhelpers.contrib.openstack.ip import resolve_address from charmhelpers.core.host import list_nics, get_nic_hwaddr @@ -175,24 +172,27 @@ class NetworkServiceContext(OSContextGenerator): return ctxt +SHARED_SECRET = "/etc/neutron/secret.txt" + + +def get_shared_secret(): + secret = None + if not os.path.exists(SHARED_SECRET): + secret = str(uuid.uuid4()) + with open(SHARED_SECRET, 'w') as secret_file: + secret_file.write(secret) + else: + with open(SHARED_SECRET, 'r') as secret_file: + secret = secret_file.read().strip() + return secret + + class DVRSharedSecretContext(OSContextGenerator): - def get_shared_secret(self): - secret = None - if not os.path.exists(self.SHARED_SECRET): - secret = str(uuid.uuid4()) - with open(self.SHARED_SECRET, 'w') as secret_file: - secret_file.write(secret) - else: - with open(self.SHARED_SECRET, 'r') as secret_file: - secret = secret_file.read().strip() - return secret - def __call__(self): - self.SHARED_SECRET = "/etc/neutron/secret.txt" if use_dvr(): ctxt = { - 'shared_secret': self.get_shared_secret(), + 'shared_secret': get_shared_secret(), 'local_ip': resolve_address(), } else: diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 8ce3d5d0..c7ed9a02 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -51,6 +51,7 @@ def config_changed(): configure_ovs() CONFIGS.write_all() + @hooks.hook('neutron-plugin-api-relation-changed') @restart_on_change(restart_map()) def neutron_plugin_api_changed(): @@ -63,6 +64,7 @@ def neutron_plugin_api_changed(): for rid in relation_ids('neutron-plugin'): neutron_plugin_joined(relation_id=rid) + @hooks.hook('neutron-plugin-relation-joined') def neutron_plugin_joined(relation_id=None): rel_data = { diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 80bd363d..5de00d0f 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -1,5 +1,6 @@ from test_utils import CharmTestCase +from test_utils import patch_open from mock import patch import neutron_ovs_context as context import charmhelpers @@ -7,6 +8,7 @@ TO_PATCH = [ 'relation_get', 'relation_ids', 'related_units', + 'resolve_address', 'config', 'unit_get', 'add_bridge', @@ -169,6 +171,7 @@ class OVSPluginContextTest(CharmTestCase): self.assertEquals(expect, napi_ctxt()) self.service_start.assertCalled() + class L3AgentContextTest(CharmTestCase): def setUp(self): @@ -196,3 +199,81 @@ class L3AgentContextTest(CharmTestCase): 'l2-population': 'True', 'overlay-network-type': 'vxlan'}) self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'legacy'}) + + +class NetworkServiceContext(CharmTestCase): + + def setUp(self): + super(NetworkServiceContext, self).setUp(context, TO_PATCH) + self.relation_get.side_effect = self.test_relation.get + self.config.side_effect = self.test_config.get + + def tearDown(self): + super(NetworkServiceContext, self).tearDown() + + def test_network_svc_ctxt(self): + self.related_units.return_value = ['unit1'] + self.relation_ids.return_value = ['rid2'] + self.test_relation.set({'service_protocol': 'http', + 'keystone_host': '10.0.0.10', + 'service_port': '8080', + 'region': 'region1', + 'service_tenant': 'tenant', + 'service_username': 'bob', + 'service_password': 'reallyhardpass'}) + self.assertEquals(context.NetworkServiceContext()(), + {'service_protocol': 'http', + 'keystone_host': '10.0.0.10', + 'service_port': '8080', + 'region': 'region1', + 'service_tenant': 'tenant', + 'service_username': 'bob', + 'service_password': 'reallyhardpass'}) + + +class DVRSharedSecretContext(CharmTestCase): + + def setUp(self): + super(DVRSharedSecretContext, self).setUp(context, + TO_PATCH) + self.config.side_effect = self.test_config.get + + @patch('os.path') + @patch('uuid.uuid4') + def test_secret_created_stored(self, _uuid4, _path): + _path.exists.return_value = False + _uuid4.return_value = 'secret_thing' + with patch_open() as (_open, _file): + self.assertEquals(context.get_shared_secret(), + 'secret_thing') + _open.assert_called_with( + context.SHARED_SECRET.format('quantum'), 'w') + _file.write.assert_called_with('secret_thing') + + @patch('os.path') + def test_secret_retrieved(self, _path): + _path.exists.return_value = True + with patch_open() as (_open, _file): + _file.read.return_value = 'secret_thing\n' + self.assertEquals(context.get_shared_secret(), + 'secret_thing') + _open.assert_called_with( + context.SHARED_SECRET.format('quantum'), 'r') + + @patch.object(context, 'use_dvr') + @patch.object(context, 'get_shared_secret') + def test_shared_secretcontext_dvr(self, _shared_secret, _use_dvr): + _shared_secret.return_value = 'secret_thing' + _use_dvr.return_value = True + self.resolve_address.return_value = '10.0.0.10' + self.assertEquals(context.DVRSharedSecretContext()(), + {'shared_secret': 'secret_thing', + 'local_ip': '10.0.0.10'}) + + @patch.object(context, 'use_dvr') + @patch.object(context, 'get_shared_secret') + def test_shared_secretcontext_nodvr(self, _shared_secret, _use_dvr): + _shared_secret.return_value = 'secret_thing' + _use_dvr.return_value = False + self.resolve_address.return_value = '10.0.0.10' + self.assertEquals(context.DVRSharedSecretContext()(), {}) diff --git a/unit_tests/test_neutron_ovs_hooks.py b/unit_tests/test_neutron_ovs_hooks.py index ac6317bf..c4180a98 100644 --- a/unit_tests/test_neutron_ovs_hooks.py +++ b/unit_tests/test_neutron_ovs_hooks.py @@ -24,7 +24,10 @@ TO_PATCH = [ 'config', 'CONFIGS', 'determine_packages', + 'determine_dvr_packages', + 'get_shared_secret', 'log', + 'relation_ids', 'relation_set', 'configure_ovs', ] @@ -61,6 +64,39 @@ class NeutronOVSHooksTests(CharmTestCase): self.assertTrue(self.CONFIGS.write_all.called) self.configure_ovs.assert_called_with() + @patch.object(neutron_ovs_context, 'use_dvr') + def test_config_changed_dvr(self, _use_dvr): + _use_dvr.return_value = True + self.determine_dvr_packages.return_value = ['dvr'] + self._call_hook('config-changed') + self.apt_update.assert_called_with() + self.assertTrue(self.CONFIGS.write_all.called) + self.apt_install.assert_has_calls([ + call(['dvr'], fatal=True), + ]) + self.configure_ovs.assert_called_with() + + @patch.object(hooks, 'neutron_plugin_joined') + @patch.object(neutron_ovs_context, 'use_dvr') + def test_neutron_plugin_api(self, _use_dvr, _plugin_joined): + _use_dvr.return_value = False + self.relation_ids.return_value = ['rid'] + self._call_hook('neutron-plugin-api-relation-changed') + self.configure_ovs.assert_called_with() + self.assertTrue(self.CONFIGS.write_all.called) + _plugin_joined.assert_called_with(relation_id='rid') + + def test_neutron_plugin_joined(self): + self.get_shared_secret.return_value = 'secret' + self._call_hook('neutron-plugin-relation-joined') + rel_data = { + 'metadata-shared-secret': 'secret', + } + self.relation_set.assert_called_with( + relation_id=None, + **rel_data + ) + def test_amqp_joined(self): self._call_hook('amqp-relation-joined') self.relation_set.assert_called_with( diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index ceee90cb..6ed7f357 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -1,7 +1,8 @@ -from mock import MagicMock, patch +from mock import MagicMock, patch, call from collections import OrderedDict import charmhelpers.contrib.openstack.templating as templating +from charmhelpers.contrib.openstack import context templating.OSConfigRenderer = MagicMock() @@ -16,6 +17,8 @@ import charmhelpers.core.hookenv as hookenv TO_PATCH = [ + 'add_bridge', + 'add_bridge_port', 'os_release', 'neutron_plugin_attribute', ] @@ -39,6 +42,15 @@ def _mock_npa(plugin, attr, net_manager=None): return plugins[plugin][attr] +class DummyContext(): + + def __init__(self, return_value): + self.return_value = return_value + + def __call__(self): + return self.return_value + + class TestNeutronOVSUtils(CharmTestCase): def setUp(self): @@ -83,8 +95,20 @@ class TestNeutronOVSUtils(CharmTestCase): def test_resource_map(self, _use_dvr): _use_dvr.return_value = False _map = nutils.resource_map() + svcs = ['neutron-plugin-openvswitch-agent'] confs = [nutils.NEUTRON_CONF] [self.assertIn(q_conf, _map.keys()) for q_conf in confs] + self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) + + @patch.object(neutron_ovs_context, 'use_dvr') + def test_resource_map_dvr(self, _use_dvr): + _use_dvr.return_value = True + _map = nutils.resource_map() + svcs = ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent', + 'neutron-vpn-agent'] + confs = [nutils.NEUTRON_CONF] + [self.assertIn(q_conf, _map.keys()) for q_conf in confs] + self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) @patch.object(neutron_ovs_context, 'use_dvr') def test_restart_map(self, _use_dvr): @@ -99,3 +123,21 @@ class TestNeutronOVSUtils(CharmTestCase): for item in _restart_map: self.assertTrue(item in _restart_map) self.assertTrue(expect[item] == _restart_map[item]) + + @patch.object(context, 'ExternalPortContext') + def test_configure_ovs_ovs_ext_port(self, _ext_port_ctxt): + _ext_port_ctxt.return_value = \ + DummyContext(return_value={'ext_port': 'eth0'}) + nutils.configure_ovs() + self.add_bridge.assert_has_calls([ + call('br-int'), + call('br-ex'), + call('br-data') + ]) + self.add_bridge_port.assert_called_with('br-ex', 'eth0') + + @patch.object(neutron_ovs_context, 'DVRSharedSecretContext') + def test_get_shared_secret(self, _dvr_secret_ctxt): + _dvr_secret_ctxt.return_value = \ + DummyContext(return_value={'shared_secret': 'supersecret'}) + self.assertEqual(nutils.get_shared_secret(), 'supersecret') From 7a4e0e1222f4dcadb0d2f556c5b3f76184ba604d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 2 Mar 2015 14:56:11 +0000 Subject: [PATCH 37/61] Purge pkgs when not in dvr mode --- hooks/neutron_ovs_hooks.py | 13 +++++++++---- hooks/neutron_ovs_utils.py | 14 ++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index c7ed9a02..c5201352 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -16,16 +16,18 @@ from charmhelpers.core.host import ( ) from charmhelpers.fetch import ( - apt_install, apt_update + apt_install, apt_update, apt_purge ) from neutron_ovs_utils import ( + DVR_PACKAGES, configure_ovs, determine_packages, determine_dvr_packages, get_shared_secret, register_configs, restart_map, + use_dvr, ) hooks = Hooks() @@ -55,9 +57,11 @@ def config_changed(): @hooks.hook('neutron-plugin-api-relation-changed') @restart_on_change(restart_map()) def neutron_plugin_api_changed(): - if determine_dvr_packages(): + if use_dvr(): apt_update() - apt_install(determine_dvr_packages(), fatal=True) + apt_install(DVR_PACKAGES, fatal=True) + else: + apt_purge(DVR_PACKAGES, fatal=True) configure_ovs() CONFIGS.write_all() # If dvr setting has changed, need to pass that on @@ -67,8 +71,9 @@ def neutron_plugin_api_changed(): @hooks.hook('neutron-plugin-relation-joined') def neutron_plugin_joined(relation_id=None): + secret = get_shared_secret() if use_dvr() else None rel_data = { - 'metadata-shared-secret': get_shared_secret() + 'metadata-shared-secret': secret, } relation_set(relation_id=relation_id, **rel_data) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index f64d58ff..55a8c8d7 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -21,7 +21,7 @@ NEUTRON_FWAAS_CONF = "/etc/neutron/fwaas_driver.ini" ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR EXT_PORT_CONF = '/etc/init/ext-port.conf' NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini" - +DVR_PACKAGES = ['neutron-vpn-agent'] BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { @@ -60,10 +60,9 @@ DATA_BRIDGE = 'br-data' def determine_dvr_packages(): - pkgs = [] - if neutron_ovs_context.use_dvr(): - pkgs = ['neutron-vpn-agent'] - return pkgs + if use_dvr(): + return DVR_PACKAGES + return [] def determine_packages(): @@ -87,7 +86,7 @@ def resource_map(): hook execution. ''' resource_map = deepcopy(BASE_RESOURCE_MAP) - if neutron_ovs_context.use_dvr(): + if use_dvr(): resource_map.update(DVR_RESOURCE_MAP) dvr_services = ['neutron-metadata-agent', 'neutron-vpn-agent'] resource_map[NEUTRON_CONF]['services'] += dvr_services @@ -116,3 +115,6 @@ def get_shared_secret(): ctxt = neutron_ovs_context.DVRSharedSecretContext()() if 'shared_secret' in ctxt: return ctxt['shared_secret'] + +def use_dvr(): + return neutron_ovs_context.use_dvr() From 8534a357afc2d8ed263d4f85512c1a0e1c2e8c47 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Mar 2015 11:12:19 +0000 Subject: [PATCH 38/61] Fix lint and unit tests --- hooks/neutron_ovs_utils.py | 1 + unit_tests/test_neutron_ovs_hooks.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 55a8c8d7..252656a0 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -116,5 +116,6 @@ def get_shared_secret(): if 'shared_secret' in ctxt: return ctxt['shared_secret'] + def use_dvr(): return neutron_ovs_context.use_dvr() diff --git a/unit_tests/test_neutron_ovs_hooks.py b/unit_tests/test_neutron_ovs_hooks.py index c4180a98..dfa7d29d 100644 --- a/unit_tests/test_neutron_ovs_hooks.py +++ b/unit_tests/test_neutron_ovs_hooks.py @@ -21,6 +21,7 @@ utils.restart_map = _map TO_PATCH = [ 'apt_update', 'apt_install', + 'apt_purge', 'config', 'CONFIGS', 'determine_packages', @@ -30,6 +31,7 @@ TO_PATCH = [ 'relation_ids', 'relation_set', 'configure_ovs', + 'use_dvr', ] NEUTRON_CONF_DIR = "/etc/neutron" From 0b2933c5a1c245bcb8c0142a9ae69d5176c331cf Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 7 Mar 2015 21:31:05 -0500 Subject: [PATCH 39/61] Sync charm-helpers --- hooks/charmhelpers/contrib/network/ip.py | 85 ++++++- .../charmhelpers/contrib/openstack/context.py | 72 +++++- hooks/charmhelpers/contrib/openstack/utils.py | 208 +++++++----------- hooks/charmhelpers/core/services/helpers.py | 16 +- 4 files changed, 243 insertions(+), 138 deletions(-) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index c7c4cd4a..2d9a95cd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from base64 import b64decode from subprocess import check_call import six +import yaml from charmhelpers.fetch import ( apt_install, @@ -104,9 +105,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -191,7 +224,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: @@ -279,9 +312,25 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): interfaces = ['identity-service'] + def __init__(self, service=None, service_user=None): + self.service = service + self.service_user = service_user + def __call__(self): log('Generating template context for identity-service', level=DEBUG) ctxt = {} + + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + ctxt['signing_dir'] = cachedir + for rid in relation_ids('identity-service'): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) @@ -291,15 +340,16 @@ class IdentityServiceContext(OSContextGenerator): auth_host = format_ipv6_addr(auth_host) or auth_host svc_protocol = rdata.get('service_protocol') or 'http' auth_protocol = rdata.get('auth_protocol') or 'http' - ctxt = {'service_port': rdata.get('service_port'), - 'service_host': serv_host, - 'auth_host': auth_host, - 'auth_port': rdata.get('auth_port'), - 'admin_tenant_name': rdata.get('service_tenant'), - 'admin_user': rdata.get('service_username'), - 'admin_password': rdata.get('service_password'), - 'service_protocol': svc_protocol, - 'auth_protocol': auth_protocol} + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol}) + if context_complete(ctxt): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs @@ -1021,6 +1071,8 @@ class ZeroMQContext(OSContextGenerator): for unit in related_units(rid): ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..0293c7d7 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -20,15 +20,18 @@ from collections import OrderedDict from functools import wraps +import errno import subprocess import json import os -import socket import sys +import time import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -421,77 +424,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): @@ -536,89 +472,115 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != "None" + return config('openstack-origin-git') != None requirements_dir = None -def git_clone_and_install(file_name, core_project): - """Clone/install all OpenStack repos specified in yaml config file.""" +def git_clone_and_install(projects, core_project, + parent_dir='/mnt/openstack-git'): + """Clone/install all OpenStack repos specified in projects dictionary.""" global requirements_dir + update_reqs = True - if file_name == "None": + if not projects: return - yaml_file = os.path.join(charm_dir(), file_name) - # clone/install the requirements project first - installed = _git_clone_and_install_subset(yaml_file, + installed = _git_clone_and_install_subset(projects, parent_dir, whitelist=['requirements']) if 'requirements' not in installed: - error_out('requirements git repository must be specified') + update_reqs = False # clone/install all other projects except requirements and the core project blacklist = ['requirements', core_project] - _git_clone_and_install_subset(yaml_file, blacklist=blacklist, - update_requirements=True) + _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, + update_requirements=update_reqs) # clone/install the core project whitelist = [core_project] - installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist, - update_requirements=True) + installed = _git_clone_and_install_subset(projects, parent_dir, + whitelist=whitelist, + update_requirements=update_reqs) if core_project not in installed: error_out('{} git repository must be specified'.format(core_project)) -def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[], - update_requirements=False): - """Clone/install subset of OpenStack repos specified in yaml config file.""" +def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], + blacklist=[], update_requirements=False): + """Clone/install subset of OpenStack repos specified in projects dict.""" global requirements_dir installed = [] - with open(yaml_file, 'r') as fd: - projects = yaml.load(fd) - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, - update_requirements) - if proj == 'requirements': - requirements_dir = repo_dir - installed.append(proj) + for proj, val in projects.items(): + # The project subset is chosen based on the following 3 rules: + # 1) If project is in blacklist, we don't clone/install it, period. + # 2) If whitelist is empty, we clone/install everything else. + # 3) If whitelist is not empty, we clone/install everything in the + # whitelist. + if proj in blacklist: + continue + if whitelist and proj not in whitelist: + continue + repo = val['repository'] + branch = val['branch'] + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements) + if proj == 'requirements': + requirements_dir = repo_dir + installed.append(proj) return installed -def _git_clone_and_install_single(repo, branch, update_requirements=False): +def _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False): """Clone and install a single git repository.""" - dest_parent_dir = "/mnt/openstack-git/" - dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo)) + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - if not os.path.exists(dest_parent_dir): - juju_log('Host dir not mounted at {}. ' - 'Creating directory there instead.'.format(dest_parent_dir)) - os.mkdir(dest_parent_dir) - - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch) + # Note(coreycb): The parent directory for storing git repositories can be + # shared by multiple charms via bind mount, etc, so we use exception + # handling to ensure the test for existence and mkdir are atomic. + try: + os.mkdir(parent_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + pass else: - repo_dir = dest_dir + juju_log('Host directory not mounted at {}. ' + 'Directory created.'.format(parent_dir)) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) + # Note(coreycb): Similar to above, the cloned git repositories can be shared + # by multiple charms via bind mount, etc, so we use exception handling and + # special lock directories to ensure that a repository clone is only + # attempted once. + try: + os.mkdir(lock_dir) + except OSError as e: + if e.errno == errno.EEXIST: + juju_log('Lock directory exists at {}. Skip git clone and wait ' + 'for lock removal before installing.'.format(lock_dir)) + while os.path.exists(lock_dir): + juju_log('Waiting for git clone to complete before installing.') + time.sleep(1) + pass + else: + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) + + os.rmdir(lock_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..15b21664 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + super(HttpRelation).__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + super(HttpRelation).__init__(self, *args, **kwargs) def provide_data(self): return { From c8885bba2afbe0ba7323f0b84c6dd2e1a120ff29 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sun, 8 Mar 2015 10:52:40 +0000 Subject: [PATCH 40/61] Initial support for deploying from git --- README.md | 95 +++++++++++++ charm-helpers-sync.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 20 +++ hooks/neutron_ovs_hooks.py | 8 ++ hooks/neutron_ovs_utils.py | 130 +++++++++++++++++- templates/upstart/neutron-ovs-cleanup.upstart | 17 +++ .../neutron-plugin-openvswitch-agent.upstart | 18 +++ tests/basic_deployment.py | 6 +- 9 files changed, 296 insertions(+), 4 deletions(-) create mode 100644 templates/upstart/neutron-ovs-cleanup.upstart create mode 100644 templates/upstart/neutron-plugin-openvswitch-agent.upstart diff --git a/README.md b/README.md index 6318fa44..b4022f6a 100644 --- a/README.md +++ b/README.md @@ -41,3 +41,98 @@ This charm has a configuration option to allow users to disable any per-instance ... These compute nodes could then be accessed by cloud users via use of host aggregates with specific flavors to target instances to hypervisors with no per-instance security. + +# Deploying from source + +The minimal openstack-origin-git config required to deploy from source is: + + openstack-origin-git: + "{'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'stable/icehouse'}}" + +If you specify a 'requirements' repository, it will be used to update the +requirements.txt files of all other git repos that it applies to, before +they are installed: + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" + +Note that there are only two key values the charm knows about for the outermost +dictionary: 'neutron' and 'requirements'. These repositories must correspond to +these keys. If the requirements repository is specified, it will be installed +first. The neutron repository is always installed last. All other repostories +will be installed in between. + +NOTE(coreycb): The following is temporary to keep track of the full list of +current tip repos (may not be up-to-date). + + openstack-origin-git: + "{'requirements': + {'repository': 'git://git.openstack.org/openstack/requirements.git', + 'branch': 'master'}, + 'neutron-fwaas': + {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', + 'branch': 'master'}, + 'neutron-lbaas': + {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', + 'branch': 'master'}, + 'neutron-vpnaas': + {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', + 'branch': 'master'}, + 'keystonemiddleware: + {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', + 'branch: 'master'}, + 'oslo-concurrency': + {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', + 'branch: 'master'}, + 'oslo-config': + {'repository': 'git://git.openstack.org/openstack/oslo.config.git', + 'branch: 'master'}, + 'oslo-context': + {'repository': 'git://git.openstack.org/openstack/oslo.context.git', + 'branch: 'master'}, + 'oslo-db': + {'repository': 'git://git.openstack.org/openstack/oslo.db.git', + 'branch: 'master'}, + 'oslo-i18n': + {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', + 'branch: 'master'}, + 'oslo-messaging': + {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', + 'branch: 'master'}, + 'oslo-middleware: + {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git', + 'branch': 'master'}, + 'oslo-rootwrap': + {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git', + 'branch: 'master'}, + 'oslo-serialization': + {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', + 'branch: 'master'}, + 'oslo-utils': + {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', + 'branch: 'master'}, + 'pbr': + {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', + 'branch: 'master'}, + 'python-keystoneclient': + {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', + 'branch: 'master'}, + 'python-neutronclient': + {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git', + 'branch: 'master'}, + 'python-novaclient': + {'repository': 'git://git.openstack.org/openstack/python-novaclient.git', + 'branch: 'master'}, + 'stevedore': + {'repository': 'git://git.openstack.org/openstack/stevedore.git', + 'branch: 'master'}, + 'neutron': + {'repository': 'git://git.openstack.org/openstack/neutron.git', + 'branch': 'master'}}" diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 9b5e79e9..e9b66fc7 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index 48b12f6f..aaa21c31 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,4 +1,5 @@ -branch: lp:charm-helpers +#branch: lp:charm-helpers +branch: /home/corey/src/charms/git/charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index ea3ed841..6a5ff908 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,24 @@ options: + openstack-origin-git: + default: None + type: string + description: | + Specifies a YAML-formatted two-dimensional array listing the git + repositories and branches from which to install OpenStack and its + dependencies. + + When openstack-origin-git is specified, openstack-specific + packages will be installed from source rather than from the + the nova-compute charm's openstack-origin repository. + + Note that the installed config files will be determined based on + the OpenStack release of the nova-compute charm's openstack-origin + option. + + Note also that this option is processed for the initial install + only. Setting this option after deployment is not supported. + + For more details see README.md. rabbit-user: default: neutron type: string diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index eb53094d..61ce2e14 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -20,6 +20,7 @@ from charmhelpers.fetch import ( from neutron_ovs_utils import ( determine_packages, + git_install, register_configs, restart_map, ) @@ -35,6 +36,13 @@ def install(): for pkg in pkgs: apt_install(pkg, fatal=True) + # NOTE(coreycb): This is temporary for sstack proxy, unless we decide + # we need to code proxy support into the charms. + os.environ["http_proxy"] = "http://squid.internal:3128" + os.environ["https_proxy"] = "https://squid.internal:3128" + + git_install(config('openstack-origin-git')) + @hooks.hook('neutron-plugin-relation-changed') @hooks.hook('neutron-plugin-api-relation-changed') diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index b5d742de..86e24127 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -1,3 +1,7 @@ +import os +import shutil +import yaml + from charmhelpers.contrib.openstack.neutron import neutron_plugin_attribute from copy import deepcopy @@ -8,6 +12,23 @@ from charmhelpers.contrib.openstack.utils import ( ) import neutron_ovs_context +BASE_GIT_PACKAGES = [ + 'libxml2-dev', + 'libxslt1-dev', + 'openvswitch-switch', + 'python-dev', + 'python-pip', + 'python-setuptools', + 'zlib1g-dev', +] + +# ubuntu packages that should not be installed when deploying from git +GIT_PACKAGE_BLACKLIST = [ + 'neutron-server', + 'neutron-plugin-openvswitch', + 'neutron-plugin-openvswitch-agent', +] + NOVA_CONF_DIR = "/etc/nova" NEUTRON_CONF_DIR = "/etc/neutron" NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR @@ -29,7 +50,15 @@ TEMPLATES = 'templates/' def determine_packages(): - return neutron_plugin_attribute('ovs', 'packages', 'neutron') + pkgs = neutron_plugin_attribute('ovs', 'packages', 'neutron') + + if git_install_requested(): + pkgs.extend(BASE_GIT_PACKAGES) + # don't include packages that will be installed from git + for p in GIT_PACKAGE_BLACKLIST: + packages.remove(p) + + return pkgs def register_configs(release=None): @@ -56,3 +85,102 @@ def restart_map(): state. ''' return {k: v['services'] for k, v in resource_map().iteritems()} + + +def git_install(projects): + """Perform setup, and install git repos specified in yaml parameter.""" + if git_install_requested(): + git_pre_install() + git_clone_and_install(yaml.load(projects), core_project='neutron') + git_post_install() + + +def git_pre_install(): + """Perform pre-install setup.""" + dirs = [ + '/etc/neutron', + '/etc/neutron/rootwrap.d', + '/etc/neutron/plugins', + '/var/lib/neutron', + '/var/lib/neutron/lock', + '/var/log/neutron', + ] + + logs = [ + '/var/log/neutron/server.log', + ] + + adduser('neutron', shell='/bin/bash', system_user=True) + add_group('neutron', system_group=True) + add_user_to_group('neutron', 'neutron') + + for d in dirs: + mkdir(d, owner='neutron', group='neutron', perms=0700, force=False) + + for l in logs: + write_file(l, '', owner='neutron', group='neutron', perms=0600) + + +def git_post_install(): + """Perform post-install setup.""" + src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc') + configs = { + 'debug-filters': { + 'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'), + 'dest': '/etc/neutron/rootwrap.d/debug.filters', + }, + 'openvswitch-plugin-filters': { + 'src': os.path.join(src_etc, 'neutron/rootwrap.d/openvswitch-plugin.filters'), + 'dest': '/etc/neutron/rootwrap.d/openvswitch-plugin.filters', + }, + 'policy': { + 'src': os.path.join(src_etc, 'policy.json'), + 'dest': '/etc/neutron/policy.json', + }, + 'rootwrap': { + 'src': os.path.join(src_etc, 'rootwrap.conf'), + 'dest': '/etc/neutron/rootwrap.conf', + }, + } + + for conf, files in configs.iteritems(): + shutil.copyfile(files['src'], files['dest']) + + configs_dir = { + 'ml2': { + 'src': os.path.join(src_etc, 'neutron/plugins/ml2/'), + 'dest': '/etc/neutron/plugins/ml2/', + }, + } + + for conf, dirs in configs_dir.iteritems(): + shutil.copytree(dirs['src'], dirs['dest']) + + #render('neutron-server.default', '/etc/default/neutron-server', {}, perms=0o440) + #render('neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) + + neutron_ovs_agent_context = { + 'service_description': 'Neutron OpenvSwitch Plugin Agent', + 'charm_name': 'neutron-openvswitch', + 'process_name': 'neutron-openvswitch-agent', + 'cleanup_process_name': 'neutron-ovs-cleanup', + 'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini', + 'log_file': '/var/log/neutron/openvswitch-agent.log', + } + + neutron_ovs_cleanup_context = { + 'service_description': 'Neutron OpenvSwitch Cleanup', + 'charm_name': 'neutron-openvswitch', + 'process_name': 'neutron-ovs-cleanup', + 'log_file': '/var/log/neutron/ovs-cleanup.log', + } + + render('upstart/neutron-plugin-openvswitch-agent.upstart', + '/etc/init/neutron-plugin-openvswitch-agent.conf', + neutron_ovs_agent_context, perms=0o644) + + render('upstart/neutron-ovs-cleanup.upstart', + '/etc/init/neutron-ovs-cleanup.conf', + neutron_ovs_cleanup_context, perms=0o644) + + service_start('neutron-plugin-openvswitch-agent') diff --git a/templates/upstart/neutron-ovs-cleanup.upstart b/templates/upstart/neutron-ovs-cleanup.upstart new file mode 100644 index 00000000..b9221d0d --- /dev/null +++ b/templates/upstart/neutron-ovs-cleanup.upstart @@ -0,0 +1,17 @@ +description "{{ service_description }}" +author "Juju {{ charm_name }} Charm " + +start on started openvswitch-switch +stop on runlevel [!2345] + +pre-start script + mkdir -p /var/run/neutron + chown neutron:root /var/run/neutron +end script + +pre-start script + [ ! -x /usr/bin/{{ process_name }} ] && exit 0- + start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ + --log-file /var/log/neutron/{{ log_file }} \ + --config-file /etc/neutron/neutron.conf --verbose +end script diff --git a/templates/upstart/neutron-plugin-openvswitch-agent.upstart b/templates/upstart/neutron-plugin-openvswitch-agent.upstart new file mode 100644 index 00000000..17a82edb --- /dev/null +++ b/templates/upstart/neutron-plugin-openvswitch-agent.upstart @@ -0,0 +1,18 @@ +description "{{ service_description }}" +author "Juju {{ charm_name }} Charm " + +start on runlevel [2345] and started {{ cleanup_process_name}} +stop on runlevel [!2345] + +respawn + +chdir /var/run + +pre-start script + mkdir -p /var/run/neutron + chown neutron:root /var/run/neutron +end script + +exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ + --config-file=/etc/neutron/neutron.conf --config-file={{ plugin_config }} \ + --log-file={{ log_file }} diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index 6fee544d..b0e83403 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -61,7 +61,11 @@ class NeutronOVSBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): """Configure all of the services.""" - configs = {} + neutron_ovs_config = {'openstack-origin-git': + "{'neutron':" + " {'repository': 'git://git.openstack.org/openstack/neutron.git'," + " 'branch': 'stable/icehouse'}}"} + configs = {'neutron-openvswitch': neutron_ovs_config} super(NeutronOVSBasicDeployment, self)._configure_services(configs) def _initialize_tests(self): From b30df45a48c542e8a3d63b6bff4874ffbbb169d5 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 16 Mar 2015 16:15:06 +0000 Subject: [PATCH 41/61] Make 0mq support >= kilo --- hooks/neutron_ovs_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index e78a8daf..0ab3cdbc 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -70,7 +70,7 @@ def amqp_changed(): @hooks.hook('zeromq-configuration-relation-joined') -@os_requires_version('juno', 'neutron-common') +@os_requires_version('kilo', 'neutron-common') def zeromq_configuration_relation_joined(relid=None): relation_set(relation_id=relid, topics=" ".join(get_topics()), From c2aa14e89d44262bd31f801078295d1b04565ec7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 09:50:10 +0000 Subject: [PATCH 42/61] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 27 +- .../charmhelpers/contrib/openstack/context.py | 60 ++++- .../charmhelpers/contrib/openstack/neutron.py | 70 ++++++ hooks/charmhelpers/contrib/openstack/utils.py | 234 ++++++++++-------- hooks/charmhelpers/core/hookenv.py | 26 ++ hooks/charmhelpers/core/host.py | 6 +- hooks/charmhelpers/core/services/helpers.py | 4 +- hooks/charmhelpers/core/unitdata.py | 2 +- .../contrib/openstack/amulet/deployment.py | 27 +- 9 files changed, 344 insertions(+), 112 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 2d9a95cd..90ac6d69 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -16,6 +16,7 @@ import json import os +import re import time from base64 import b64decode from subprocess import check_call @@ -48,6 +49,8 @@ from charmhelpers.core.hookenv import ( from charmhelpers.core.sysctl import create as sysctl_create from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr, mkdir, write_file, ) @@ -65,12 +68,18 @@ from charmhelpers.contrib.hahelpers.apache import ( from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + INTERNAL, +) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_ipv4_addr, get_ipv6_addr, get_netmask_for_address, format_ipv6_addr, is_address_in_network, + is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip @@ -727,7 +736,14 @@ class ApacheSSLContext(OSContextGenerator): 'endpoints': [], 'ext_ports': []} - for cn in self.canonical_names(): + cns = self.canonical_names() + if cns: + for cn in cns: + self.configure_cert(cn) + else: + # Expect cert/key provided in config (currently assumed that ca + # uses ip for cn) + cn = resolve_address(endpoint_type=INTERNAL) self.configure_cert(cn) addresses = self.get_network_addresses() @@ -883,6 +899,48 @@ class NeutronContext(OSContextGenerator): return ctxt +class NeutronPortContext(OSContextGenerator): + NIC_PREFIXES = ['eth', 'bond'] + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(self.NIC_PREFIXES): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + resolved = [] + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in ports: + if re.match(mac_regex, entry): + # NIC is in known NICs and does NOT hace an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + resolved.append(hwaddr_to_nic[entry]) + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + resolved.append(entry) + + return resolved + + class OSConfigFlagContext(OSContextGenerator): """Provides support for user-defined config flags. diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 902757fe..f8851050 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -16,6 +16,7 @@ # Various utilies for dealing with Neutron and the renaming from Quantum. +import six from subprocess import check_output from charmhelpers.core.hookenv import ( @@ -237,3 +238,72 @@ def network_manager(): else: # ensure accurate naming for all releases post-H return 'neutron' + + +def parse_mappings(mappings): + parsed = {} + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + parsed[p[0].strip()] = p[2].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings, default_bridge='br-data'): + """Parse data port mappings. + + Mappings must be a space-delimited list of bridge:port mappings. + + Returns dict of the form {bridge:port}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {default_bridge: mappings.split(' ')[0]} + + bridges = _mappings.keys() + ports = _mappings.values() + if len(set(bridges)) != len(bridges): + raise Exception("It is not allowed to have more than one port " + "configured on the same bridge") + + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return _mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + return {} + + mappings = {} + for p, r in six.iteritems(_mappings): + mappings[p] = tuple(r.split(':')) + + return mappings diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0293c7d7..da65f6d3 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -20,18 +20,21 @@ from collections import OrderedDict from functools import wraps -import errno import subprocess import json import os import sys -import time import six import yaml from charmhelpers.contrib.network import ip +from charmhelpers.core import ( + hookenv, + unitdata, +) + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -332,6 +335,21 @@ def configure_installation_source(rel): error_out("Invalid openstack-release specified: %s" % rel) +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = hookenv.execution_environment()['conf'][option] + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + def save_script_rc(script_path="scripts/scriptrc", **env_vars): """ Write an rc file in the charm-delivered directory containing @@ -471,116 +489,103 @@ def os_requires_version(ostack_release, pkg): def git_install_requested(): - """Returns true if openstack-origin-git is specified.""" - return config('openstack-origin-git') != None + """ + Returns true if openstack-origin-git is specified. + """ + return config('openstack-origin-git').lower() != "none" requirements_dir = None -def git_clone_and_install(projects, core_project, - parent_dir='/mnt/openstack-git'): - """Clone/install all OpenStack repos specified in projects dictionary.""" - global requirements_dir - update_reqs = True +def git_clone_and_install(projects_yaml, core_project): + """ + Clone/install all specified OpenStack repositories. - if not projects: + The expected format of projects_yaml is: + repositories: + - {name: keystone, + repository: 'git://git.openstack.org/openstack/keystone.git', + branch: 'stable/icehouse'} + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements.git', + branch: 'stable/icehouse'} + directory: /mnt/openstack-git + + The directory key is optional. + """ + global requirements_dir + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: return - # clone/install the requirements project first - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=['requirements']) - if 'requirements' not in installed: - update_reqs = False + projects = yaml.load(projects_yaml) + _git_validate_projects_yaml(projects, core_project) - # clone/install all other projects except requirements and the core project - blacklist = ['requirements', core_project] - _git_clone_and_install_subset(projects, parent_dir, blacklist=blacklist, - update_requirements=update_reqs) + if 'directory' in projects.keys(): + parent_dir = projects['directory'] - # clone/install the core project - whitelist = [core_project] - installed = _git_clone_and_install_subset(projects, parent_dir, - whitelist=whitelist, - update_requirements=update_reqs) - if core_project not in installed: - error_out('{} git repository must be specified'.format(core_project)) - - -def _git_clone_and_install_subset(projects, parent_dir, whitelist=[], - blacklist=[], update_requirements=False): - """Clone/install subset of OpenStack repos specified in projects dict.""" - global requirements_dir - installed = [] - - for proj, val in projects.items(): - # The project subset is chosen based on the following 3 rules: - # 1) If project is in blacklist, we don't clone/install it, period. - # 2) If whitelist is empty, we clone/install everything else. - # 3) If whitelist is not empty, we clone/install everything in the - # whitelist. - if proj in blacklist: - continue - if whitelist and proj not in whitelist: - continue - repo = val['repository'] - branch = val['branch'] - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements) - if proj == 'requirements': + for p in projects['repositories']: + repo = p['repository'] + branch = p['branch'] + if p['name'] == 'requirements': + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=False) requirements_dir = repo_dir - installed.append(proj) - return installed - - -def _git_clone_and_install_single(repo, branch, parent_dir, - update_requirements=False): - """Clone and install a single git repository.""" - dest_dir = os.path.join(parent_dir, os.path.basename(repo)) - lock_dir = os.path.join(parent_dir, os.path.basename(repo) + '.lock') - - # Note(coreycb): The parent directory for storing git repositories can be - # shared by multiple charms via bind mount, etc, so we use exception - # handling to ensure the test for existence and mkdir are atomic. - try: - os.mkdir(parent_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Directory already exists at {}. ' - 'No need to create directory.'.format(parent_dir)) - pass - else: - juju_log('Host directory not mounted at {}. ' - 'Directory created.'.format(parent_dir)) - - # Note(coreycb): Similar to above, the cloned git repositories can be shared - # by multiple charms via bind mount, etc, so we use exception handling and - # special lock directories to ensure that a repository clone is only - # attempted once. - try: - os.mkdir(lock_dir) - except OSError as e: - if e.errno == errno.EEXIST: - juju_log('Lock directory exists at {}. Skip git clone and wait ' - 'for lock removal before installing.'.format(lock_dir)) - while os.path.exists(lock_dir): - juju_log('Waiting for git clone to complete before installing.') - time.sleep(1) - pass - else: - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) else: - repo_dir = dest_dir + repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + update_requirements=True) - if update_requirements: - if not requirements_dir: - error_out('requirements repo must be cloned before ' - 'updating from global requirements.') - _git_update_requirements(repo_dir, requirements_dir) - os.rmdir(lock_dir) +def _git_validate_projects_yaml(projects, core_project): + """ + Validate the projects yaml. + """ + _git_ensure_key_exists('repositories', projects) + + for project in projects['repositories']: + _git_ensure_key_exists('name', project.keys()) + _git_ensure_key_exists('repository', project.keys()) + _git_ensure_key_exists('branch', project.keys()) + + if projects['repositories'][0]['name'] != 'requirements': + error_out('{} git repo must be specified first'.format('requirements')) + + if projects['repositories'][-1]['name'] != core_project: + error_out('{} git repo must be specified last'.format(core_project)) + + +def _git_ensure_key_exists(key, keys): + """ + Ensure that key exists in keys. + """ + if key not in keys: + error_out('openstack-origin-git key \'{}\' is missing'.format(key)) + + +def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): + """ + Clone and install a single git repository. + """ + dest_dir = os.path.join(parent_dir, os.path.basename(repo)) + + if not os.path.exists(parent_dir): + juju_log('Directory already exists at {}. ' + 'No need to create directory.'.format(parent_dir)) + os.mkdir(parent_dir) + + if not os.path.exists(dest_dir): + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + else: + repo_dir = dest_dir + + if update_requirements: + if not requirements_dir: + error_out('requirements repo must be cloned before ' + 'updating from global requirements.') + _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) pip_install(repo_dir) @@ -589,16 +594,39 @@ def _git_clone_and_install_single(repo, branch, parent_dir, def _git_update_requirements(package_dir, reqs_dir): - """Update from global requirements. + """ + Update from global requirements. - Update an OpenStack git directory's requirements.txt and - test-requirements.txt from global-requirements.txt.""" + Update an OpenStack git directory's requirements.txt and + test-requirements.txt from global-requirements.txt. + """ orig_dir = os.getcwd() os.chdir(reqs_dir) - cmd = "python update.py {}".format(package_dir) + cmd = ['python', 'update.py', package_dir] try: - subprocess.check_call(cmd.split(' ')) + subprocess.check_call(cmd) except subprocess.CalledProcessError: package = os.path.basename(package_dir) error_out("Error updating {} from global-requirements.txt".format(package)) os.chdir(orig_dir) + + +def git_src_dir(projects_yaml, project): + """ + Return the directory where the specified project's source is located. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + for p in projects['repositories']: + if p['name'] == project: + return os.path.join(parent_dir, os.path.basename(p['repository'])) + + return None diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index cf552b39..715dd4c5 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -566,3 +566,29 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" return os.environ.get('CHARM_DIR') + + +@cached +def action_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +def action_set(values): + """Sets the values to be returned after the action finishes""" + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def action_fail(message): + """Sets the action status to failed and sets the error message. + + The results set by action_set are preserved.""" + subprocess.check_call(['action-fail', message]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b771c611..830822af 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -339,12 +339,16 @@ def lsb_release(): def pwgen(length=None): """Generate a random pasword.""" if length is None: + # A random length is ok to use a weak PRNG length = random.choice(range(35, 45)) alphanumeric_chars = [ l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() random_chars = [ - random.choice(alphanumeric_chars) for _ in range(length)] + random_generator.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 15b21664..3eb5fb44 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -139,7 +139,7 @@ class MysqlRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'user', 'password', 'database'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -154,7 +154,7 @@ class HttpRelation(RelationContext): def __init__(self, *args, **kwargs): self.required_keys = ['host', 'port'] - super(HttpRelation).__init__(self, *args, **kwargs) + RelationContext.__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py index 3000134a..406a35c5 100644 --- a/hooks/charmhelpers/core/unitdata.py +++ b/hooks/charmhelpers/core/unitdata.py @@ -443,7 +443,7 @@ class HookData(object): data = hookenv.execution_environment() self.conf = conf_delta = self.kv.delta(data['conf'], 'config') self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') - self.kv.set('env', data['env']) + self.kv.set('env', dict(data['env'])) self.kv.set('unit', data['unit']) self.kv.set('relid', data.get('relid')) return conf_delta, rels_delta diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 0cfeaa4c..0e0db566 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -15,6 +15,7 @@ # along with charm-helpers. If not, see . import six +from collections import OrderedDict from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -100,12 +101,34 @@ class OpenStackAmuletDeployment(AmuletDeployment): """ (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse) = range(6) + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo) = range(8) releases = { ('precise', None): self.precise_essex, ('precise', 'cloud:precise-folsom'): self.precise_folsom, ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, ('precise', 'cloud:precise-havana'): self.precise_havana, ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, - ('trusty', None): self.trusty_icehouse} + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo} return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] From d5e9cebd14dd452c4ed0eb0fdf41b634b109bc22 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 21 Mar 2015 10:25:07 +0000 Subject: [PATCH 43/61] Overall refresh to current install from source approach --- Makefile | 7 +- README.md | 166 +++++++++--------- actions.yaml | 2 + actions/git-reinstall | 1 + actions/git_reinstall.py | 40 +++++ ...pers-sync.yaml => charm-helpers-hooks.yaml | 3 +- charm-helpers-tests.yaml | 3 +- config.yaml | 12 +- hooks/neutron_ovs_hooks.py | 9 + hooks/neutron_ovs_utils.py | 17 +- tests/16-basic-trusty-icehouse-git | 9 + ...basic-trusty-juno => 17-basic-trusty-juno} | 0 tests/18-basic-trusty-juno-git | 12 ++ tests/basic_deployment.py | 24 ++- unit_tests/__init__.py | 2 + unit_tests/test_actions_git_reinstall.py | 85 +++++++++ 16 files changed, 279 insertions(+), 113 deletions(-) create mode 100644 actions.yaml create mode 120000 actions/git-reinstall create mode 100755 actions/git_reinstall.py rename charm-helpers-sync.yaml => charm-helpers-hooks.yaml (76%) create mode 100755 tests/16-basic-trusty-icehouse-git rename tests/{16-basic-trusty-juno => 17-basic-trusty-juno} (100%) create mode 100755 tests/18-basic-trusty-juno-git create mode 100644 unit_tests/test_actions_git_reinstall.py diff --git a/Makefile b/Makefile index db4e2c25..6ca58a67 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON := /usr/bin/env python lint: - @flake8 --exclude hooks/charmhelpers hooks unit_tests tests + @flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests @charm proof unit_test: @@ -15,7 +15,7 @@ bin/charm_helpers_sync.py: > bin/charm_helpers_sync.py sync: bin/charm_helpers_sync.py - @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml test: @@ -25,7 +25,8 @@ test: # https://bugs.launchpad.net/amulet/+bug/1320357 @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ - 16-basic-trusty-juno + 16-basic-trusty-icehouse-git 17-basic-trusty-juno \ + 18-basic-trusty-juno-git publish: lint unit_test bzr push lp:charms/neutron-openvswitch diff --git a/README.md b/README.md index b4022f6a..05d08869 100644 --- a/README.md +++ b/README.md @@ -44,95 +44,87 @@ These compute nodes could then be accessed by cloud users via use of host aggreg # Deploying from source -The minimal openstack-origin-git config required to deploy from source is: +The minimum openstack-origin-git config required to deploy from source is: openstack-origin-git: - "{'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'stable/icehouse'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: stable/juno}" -If you specify a 'requirements' repository, it will be used to update the -requirements.txt files of all other git repos that it applies to, before -they are installed: +Note that there are only two 'name' values the charm knows about: 'requirements' +and 'neutron'. These repositories must correspond to these 'name' values. +Additionally, the requirements repository must be specified first and the +neutron repository must be specified last. All other repostories are installed +in the order in which they are specified. + +The following is a full list of current tip repos (may not be up-to-date): openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'master'}}" - -Note that there are only two key values the charm knows about for the outermost -dictionary: 'neutron' and 'requirements'. These repositories must correspond to -these keys. If the requirements repository is specified, it will be installed -first. The neutron repository is always installed last. All other repostories -will be installed in between. - -NOTE(coreycb): The following is temporary to keep track of the full list of -current tip repos (may not be up-to-date). - - openstack-origin-git: - "{'requirements': - {'repository': 'git://git.openstack.org/openstack/requirements.git', - 'branch': 'master'}, - 'neutron-fwaas': - {'repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', - 'branch': 'master'}, - 'neutron-lbaas': - {'repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', - 'branch': 'master'}, - 'neutron-vpnaas': - {'repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', - 'branch': 'master'}, - 'keystonemiddleware: - {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git', - 'branch: 'master'}, - 'oslo-concurrency': - {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git', - 'branch: 'master'}, - 'oslo-config': - {'repository': 'git://git.openstack.org/openstack/oslo.config.git', - 'branch: 'master'}, - 'oslo-context': - {'repository': 'git://git.openstack.org/openstack/oslo.context.git', - 'branch: 'master'}, - 'oslo-db': - {'repository': 'git://git.openstack.org/openstack/oslo.db.git', - 'branch: 'master'}, - 'oslo-i18n': - {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git', - 'branch: 'master'}, - 'oslo-messaging': - {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git', - 'branch: 'master'}, - 'oslo-middleware: - {'repository': 'git://git.openstack.org/openstack/oslo.middleware.git', - 'branch': 'master'}, - 'oslo-rootwrap': - {'repository': 'git://git.openstack.org/openstack/oslo.rootwrap.git', - 'branch: 'master'}, - 'oslo-serialization': - {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git', - 'branch: 'master'}, - 'oslo-utils': - {'repository': 'git://git.openstack.org/openstack/oslo.utils.git', - 'branch: 'master'}, - 'pbr': - {'repository': 'git://git.openstack.org/openstack-dev/pbr.git', - 'branch: 'master'}, - 'python-keystoneclient': - {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git', - 'branch: 'master'}, - 'python-neutronclient': - {'repository': 'git://git.openstack.org/openstack/python-neutronclient.git', - 'branch: 'master'}, - 'python-novaclient': - {'repository': 'git://git.openstack.org/openstack/python-novaclient.git', - 'branch: 'master'}, - 'stevedore': - {'repository': 'git://git.openstack.org/openstack/stevedore.git', - 'branch: 'master'}, - 'neutron': - {'repository': 'git://git.openstack.org/openstack/neutron.git', - 'branch': 'master'}}" + "repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: master} + - {name: oslo-concurrency, + repository: 'git://git.openstack.org/openstack/oslo.concurrency', + branch: master} + - {name: oslo-config, + repository: 'git://git.openstack.org/openstack/oslo.config', + branch: master} + - {name: oslo-context, + repository: 'git://git.openstack.org/openstack/oslo.context.git', + branch: master} + - {name: oslo-db, + repository: 'git://git.openstack.org/openstack/oslo.db', + branch: master} + - {name: oslo-i18n, + repository: 'git://git.openstack.org/openstack/oslo.i18n', + branch: master} + - {name: oslo-messaging, + repository: 'git://git.openstack.org/openstack/oslo.messaging.git', + branch: master} + - {name: oslo-middleware, + repository': 'git://git.openstack.org/openstack/oslo.middleware.git', + branch: master} + - {name: oslo-rootwrap', + repository: 'git://git.openstack.org/openstack/oslo.rootwrap.git', + branch: master} + - {name: oslo-serialization, + repository: 'git://git.openstack.org/openstack/oslo.serialization', + branch: master} + - {name: oslo-utils, + repository: 'git://git.openstack.org/openstack/oslo.utils', + branch: master} + - {name: pbr, + repository: 'git://git.openstack.org/openstack-dev/pbr', + branch: master} + - {name: stevedore, + repository: 'git://git.openstack.org/openstack/stevedore.git', + branch: 'master'} + - {name: python-keystoneclient, + repository: 'git://git.openstack.org/openstack/python-keystoneclient', + branch: master} + - {name: python-neutronclient, + repository: 'git://git.openstack.org/openstack/python-neutronclient.git', + branch: master} + - {name: python-novaclient, + repository': 'git://git.openstack.org/openstack/python-novaclient.git', + branch: master} + - {name: keystonemiddleware, + repository: 'git://git.openstack.org/openstack/keystonemiddleware', + branch: master} + - {name: neutron-fwaas, + repository': 'git://git.openstack.org/openstack/neutron-fwaas.git', + branch: master} + - {name: neutron-lbaas, + repository: 'git://git.openstack.org/openstack/neutron-lbaas.git', + branch: master} + - {name: neutron-vpnaas, + repository: 'git://git.openstack.org/openstack/neutron-vpnaas.git', + branch: master} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: master}" diff --git a/actions.yaml b/actions.yaml new file mode 100644 index 00000000..02fbf59a --- /dev/null +++ b/actions.yaml @@ -0,0 +1,2 @@ +git-reinstall: + description: Reinstall neutron-openvswitch from the openstack-origin-git repositories. diff --git a/actions/git-reinstall b/actions/git-reinstall new file mode 120000 index 00000000..ff684984 --- /dev/null +++ b/actions/git-reinstall @@ -0,0 +1 @@ +git_reinstall.py \ No newline at end of file diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py new file mode 100755 index 00000000..849e6634 --- /dev/null +++ b/actions/git_reinstall.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +import sys +import traceback + +sys.path.append('hooks/') + +from charmhelpers.contrib.openstack.utils import ( + git_install_requested, +) + +from charmhelpers.core.hookenv import ( + action_set, + action_fail, + config, +) + +from neutron_ovs_utils import ( + git_install, +) + + +def git_reinstall(): + """Reinstall from source and restart services. + + If the openstack-origin-git config option was used to install openstack + from source git repositories, then this action can be used to reinstall + from updated git repositories, followed by a restart of services.""" + if not git_install_requested(): + action_fail('openstack-origin-git is not configured') + return + + try: + git_install(config('openstack-origin-git')) + except: + action_set({'traceback': traceback.format_exc()}) + action_fail('git-reinstall resulted in an unexpected error') + + +if __name__ == '__main__': + git_reinstall() diff --git a/charm-helpers-sync.yaml b/charm-helpers-hooks.yaml similarity index 76% rename from charm-helpers-sync.yaml rename to charm-helpers-hooks.yaml index e9b66fc7..9b5e79e9 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-hooks.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml index aaa21c31..48b12f6f 100644 --- a/charm-helpers-tests.yaml +++ b/charm-helpers-tests.yaml @@ -1,5 +1,4 @@ -#branch: lp:charm-helpers -branch: /home/corey/src/charms/git/charm-helpers +branch: lp:charm-helpers destination: tests/charmhelpers include: - contrib.amulet diff --git a/config.yaml b/config.yaml index 6a5ff908..501aff83 100644 --- a/config.yaml +++ b/config.yaml @@ -3,20 +3,16 @@ options: default: None type: string description: | - Specifies a YAML-formatted two-dimensional array listing the git - repositories and branches from which to install OpenStack and its - dependencies. + Specifies a YAML-formatted dictionary listing the git + repositories and branches from which to install OpenStack and + its dependencies. When openstack-origin-git is specified, openstack-specific packages will be installed from source rather than from the the nova-compute charm's openstack-origin repository. Note that the installed config files will be determined based on - the OpenStack release of the nova-compute charm's openstack-origin - option. - - Note also that this option is processed for the initial install - only. Setting this option after deployment is not supported. + the OpenStack release of the openstack-origin option. For more details see README.md. rabbit-user: diff --git a/hooks/neutron_ovs_hooks.py b/hooks/neutron_ovs_hooks.py index 61ce2e14..6e8264a7 100755 --- a/hooks/neutron_ovs_hooks.py +++ b/hooks/neutron_ovs_hooks.py @@ -2,6 +2,11 @@ import sys +from charmhelpers.contrib.openstack.utils import ( + config_value_changed, + git_install_requested, +) + from charmhelpers.core.hookenv import ( Hooks, UnregisteredHookError, @@ -51,6 +56,10 @@ def install(): def config_changed(): CONFIGS.write_all() + if git_install_requested() and + config_value_changed('openstack-origin-git'): + git_install(config('openstack-origin-git')) + @hooks.hook('amqp-relation-joined') def amqp_joined(relation_id=None): diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 86e24127..cd96257b 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -6,6 +6,11 @@ from charmhelpers.contrib.openstack.neutron import neutron_plugin_attribute from copy import deepcopy from charmhelpers.contrib.openstack import context, templating +from charmhelpers.contrib.openstack.utils import ( + git_install_requested, + git_clone_and_install, + git_src_dir, +) from collections import OrderedDict from charmhelpers.contrib.openstack.utils import ( os_release, @@ -87,12 +92,12 @@ def restart_map(): return {k: v['services'] for k, v in resource_map().iteritems()} -def git_install(projects): +def git_install(projects_yaml): """Perform setup, and install git repos specified in yaml parameter.""" if git_install_requested(): git_pre_install() - git_clone_and_install(yaml.load(projects), core_project='neutron') - git_post_install() + git_clone_and_install(projects_yaml, core_project='neutron') + git_post_install(projects_yaml) def git_pre_install(): @@ -121,9 +126,9 @@ def git_pre_install(): write_file(l, '', owner='neutron', group='neutron', perms=0600) -def git_post_install(): +def git_post_install(projects_yaml): """Perform post-install setup.""" - src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/neutron-api.git/etc') + src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') configs = { 'debug-filters': { 'src': os.path.join(src_etc, 'neutron/rootwrap.d/debug.filters'), @@ -175,10 +180,10 @@ def git_post_install(): 'log_file': '/var/log/neutron/ovs-cleanup.log', } + # NOTE(coreycb): Needs systemd support render('upstart/neutron-plugin-openvswitch-agent.upstart', '/etc/init/neutron-plugin-openvswitch-agent.conf', neutron_ovs_agent_context, perms=0o644) - render('upstart/neutron-ovs-cleanup.upstart', '/etc/init/neutron-ovs-cleanup.conf', neutron_ovs_cleanup_context, perms=0o644) diff --git a/tests/16-basic-trusty-icehouse-git b/tests/16-basic-trusty-icehouse-git new file mode 100755 index 00000000..de2fec89 --- /dev/null +++ b/tests/16-basic-trusty-icehouse-git @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-openvswitch git deployment on trusty-icehouse.""" + +from basic_deployment import NeutronOVSBasicDeployment + +if __name__ == '__main__': + deployment = NeutronOVSBasicDeployment(series='trusty', git=True) + deployment.run_tests() diff --git a/tests/16-basic-trusty-juno b/tests/17-basic-trusty-juno similarity index 100% rename from tests/16-basic-trusty-juno rename to tests/17-basic-trusty-juno diff --git a/tests/18-basic-trusty-juno-git b/tests/18-basic-trusty-juno-git new file mode 100755 index 00000000..618c9aa2 --- /dev/null +++ b/tests/18-basic-trusty-juno-git @@ -0,0 +1,12 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-openvswitch git deployment on trusty-juno.""" + +from basic_deployment import NeutronOVSBasicDeployment + +if __name__ == '__main__': + deployment = NeutronOVSBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno', + git=True) + deployment.run_tests() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index b0e83403..34d2472c 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -2,6 +2,7 @@ import amulet import time +import yaml from charmhelpers.contrib.openstack.amulet.deployment import ( OpenStackAmuletDeployment @@ -24,10 +25,12 @@ u = OpenStackAmuletUtils(ERROR) class NeutronOVSBasicDeployment(OpenStackAmuletDeployment): """Amulet tests on a basic neutron-openvswtich deployment.""" - def __init__(self, series, openstack=None, source=None, stable=False): + def __init__(self, series, openstack=None, source=None, git=False, + stable=False): """Deploy the entire test environment.""" super(NeutronOVSBasicDeployment, self).__init__(series, openstack, source, stable) + self.git = git self._add_services() self._add_relations() self._configure_services() @@ -61,10 +64,21 @@ class NeutronOVSBasicDeployment(OpenStackAmuletDeployment): def _configure_services(self): """Configure all of the services.""" - neutron_ovs_config = {'openstack-origin-git': - "{'neutron':" - " {'repository': 'git://git.openstack.org/openstack/neutron.git'," - " 'branch': 'stable/icehouse'}}"} + neutron_ovs_config = {} + if self.git: + branch = 'stable/' + self._get_openstack_release_string() + openstack_origin_git = { + 'repositories': [ + {'name': 'requirements', + 'repository': 'git://git.openstack.org/openstack/requirements', + 'branch': branch}, + {'name': 'neutron', + 'repository': 'git://git.openstack.org/openstack/neutron', + 'branch': branch}, + ], + 'directory': '/mnt/openstack-git', + } + neutron_api_config['openstack-origin-git'] = yaml.dump(openstack_origin_git) configs = {'neutron-openvswitch': neutron_ovs_config} super(NeutronOVSBasicDeployment, self)._configure_services(configs) diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 415b2110..43aa3614 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -1,2 +1,4 @@ import sys + +sys.path.append('actions/') sys.path.append('hooks/') diff --git a/unit_tests/test_actions_git_reinstall.py b/unit_tests/test_actions_git_reinstall.py new file mode 100644 index 00000000..802e9f52 --- /dev/null +++ b/unit_tests/test_actions_git_reinstall.py @@ -0,0 +1,85 @@ +from mock import patch + +with patch('charmhelpers.core.hookenv.config') as config: + config.return_value = 'neutron' + import neutron_ovs_utils as utils # noqa + +import git_reinstall + +from test_utils import ( + CharmTestCase +) + +TO_PATCH = [ + 'config', +] + + +openstack_origin_git = \ + """repositories: + - {name: requirements, + repository: 'git://git.openstack.org/openstack/requirements', + branch: stable/juno} + - {name: neutron, + repository: 'git://git.openstack.org/openstack/neutron', + branch: stable/juno}""" + + +class TestNeutronOVSActions(CharmTestCase): + + def setUp(self): + super(TestNeutronOVSActions, self).setUp(git_reinstall, TO_PATCH) + self.config.side_effect = self.test_config.get + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + def test_git_reinstall(self, git_install, action_fail, action_set): + self.test_config.set('openstack-origin-git', openstack_origin_git) + + git_reinstall.git_reinstall() + + git_install.assert_called_with(openstack_origin_git) + self.assertTrue(git_install.called) + self.assertFalse(action_set.called) + self.assertFalse(action_fail.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_not_configured(self, _config, git_install, + action_fail, action_set): + _config.return_value = 'none' + + git_reinstall.git_reinstall() + + msg = 'openstack-origin-git is not configured' + action_fail.assert_called_with(msg) + self.assertFalse(git_install.called) + self.assertFalse(action_set.called) + + @patch.object(git_reinstall, 'action_set') + @patch.object(git_reinstall, 'action_fail') + @patch.object(git_reinstall, 'git_install') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_git_reinstall_exception(self, _config, git_install, + action_fail, action_set): + _config.return_value = openstack_origin_git + e = OSError('something bad happened') + git_install.side_effect = e + traceback = ( + "Traceback (most recent call last):\n" + " File \"actions/git_reinstall.py\", line 33, in git_reinstall\n" + " git_install(config(\'openstack-origin-git\'))\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n" # noqa + " return _mock_self._mock_call(*args, **kwargs)\n" + " File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n" # noqa + " raise effect\n" + "OSError: something bad happened\n") + + git_reinstall.git_reinstall() + + msg = 'git-reinstall resulted in an unexpected error' + action_fail.assert_called_with(msg) + action_set.assert_called_with({'traceback': traceback}) From 78b14f6e899a6b3660690eae3fc5ee717976ac23 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 13:39:11 +0000 Subject: [PATCH 44/61] Update after some contexts moved to charm helpers --- .../charmhelpers/contrib/openstack/context.py | 145 +++++++++++++++++- hooks/neutron_ovs_context.py | 86 +---------- hooks/neutron_ovs_utils.py | 2 +- unit_tests/test_neutron_ovs_context.py | 63 +++++--- 4 files changed, 191 insertions(+), 105 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 90ac6d69..45e65790 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -47,6 +47,7 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( list_nics, @@ -67,6 +68,7 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, ) from charmhelpers.contrib.openstack.ip import ( resolve_address, @@ -82,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -1162,3 +1163,145 @@ class SysctlContext(OSContextGenerator): sysctl_create(sysctl_dict, '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 4c6febf5..46ae4e7d 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -1,11 +1,7 @@ from charmhelpers.core.hookenv import ( - relation_ids, - related_units, - relation_get, config, unit_get, ) -from charmhelpers.core.strutils import bool_from_string from charmhelpers.contrib.openstack import context from charmhelpers.core.host import ( service_running, @@ -15,68 +11,17 @@ from charmhelpers.core.host import ( from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.contrib.network.ip import get_address_in_network +from charmhelpers.contrib.openstack.context import ( + NeutronAPIContext, + DataPortContext, +) from charmhelpers.contrib.openstack.neutron import ( parse_bridge_mappings, - parse_data_port_mappings, parse_vlan_range_mappings, ) -from charmhelpers.core.host import ( - get_nic_hwaddr, -) OVS_BRIDGE = 'br-int' -def _neutron_api_settings(): - ''' - Inspects current neutron-plugin relation - ''' - neutron_settings = { - 'neutron_security_groups': False, - 'l2_population': True, - 'overlay_network_type': 'gre', - } - - for rid in relation_ids('neutron-plugin-api'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if 'l2-population' in rdata: - neutron_settings.update({ - 'l2_population': bool_from_string(rdata['l2-population']), - 'overlay_network_type': rdata['overlay-network-type'], - 'neutron_security_groups': - bool_from_string(rdata['neutron-security-groups']) - }) - - # Override with configuration if set to true - if config('disable-security-groups'): - neutron_settings['neutron_security_groups'] = False - - net_dev_mtu = rdata.get('network-device-mtu') - if net_dev_mtu: - neutron_settings['network_device_mtu'] = net_dev_mtu - - return neutron_settings - - -class DataPortContext(context.NeutronPortContext): - - def __call__(self): - ports = config('data-port') - if ports: - portmap = parse_data_port_mappings(ports) - ports = portmap.values() - resolved = self.resolve_ports(ports) - normalized = {get_nic_hwaddr(port): port for port in resolved - if port not in ports} - normalized.update({port: port for port in resolved - if port in ports}) - if resolved: - return {bridge: normalized[port] for bridge, port in - portmap.iteritems() if port in normalized.keys()} - - return None - - class OVSPluginContext(context.NeutronContext): interfaces = [] @@ -90,7 +35,9 @@ class OVSPluginContext(context.NeutronContext): @property def neutron_security_groups(self): - neutron_api_settings = _neutron_api_settings() + if config('disable-security-groups'): + return False + neutron_api_settings = NeutronAPIContext()() return neutron_api_settings['neutron_security_groups'] def _ensure_bridge(self): @@ -125,7 +72,7 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['local_ip'] = \ get_address_in_network(config('os-data-network'), get_host_ip(unit_get('private-address'))) - neutron_api_settings = _neutron_api_settings() + neutron_api_settings = NeutronAPIContext()() ovs_ctxt['neutron_security_groups'] = self.neutron_security_groups ovs_ctxt['l2_population'] = neutron_api_settings['l2_population'] ovs_ctxt['overlay_network_type'] = \ @@ -155,20 +102,3 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['vlan_ranges'] = vlan_ranges return ovs_ctxt - - -class PhyNICMTUContext(DataPortContext): - """Context used to apply settings to neutron data-port devices""" - - def __call__(self): - ctxt = {} - mappings = super(PhyNICMTUContext, self).__call__() - if mappings and mappings.values(): - ports = mappings.values() - neutron_api_settings = _neutron_api_settings() - mtu = neutron_api_settings.get('network_device_mtu') - if mtu: - ctxt['devs'] = '\\n'.join(ports) - ctxt['mtu'] = mtu - - return ctxt diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 5cc9871c..66b8a77b 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -28,7 +28,7 @@ BASE_RESOURCE_MAP = OrderedDict([ }), (PHY_NIC_MTU_CONF, { 'services': ['os-charm-phy-nic-mtu'], - 'contexts': [neutron_ovs_context.PhyNICMTUContext()], + 'contexts': [context.PhyNICMTUContext()], }), ]) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index a7da378b..5423b5ab 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -4,15 +4,13 @@ from mock import patch import neutron_ovs_context as context import charmhelpers TO_PATCH = [ - 'relation_get', - 'relation_ids', - 'related_units', 'config', 'unit_get', 'add_bridge', 'add_bridge_port', 'service_running', 'service_start', + 'service_restart', 'get_host_ip', ] @@ -21,7 +19,6 @@ class OVSPluginContextTest(CharmTestCase): def setUp(self): super(OVSPluginContextTest, self).setUp(context, TO_PATCH) - self.relation_get.side_effect = self.test_relation.get self.config.side_effect = self.test_config.get self.test_config.set('debug', True) self.test_config.set('verbose', True) @@ -30,39 +27,44 @@ class OVSPluginContextTest(CharmTestCase): def tearDown(self): super(OVSPluginContextTest, self).tearDown() + @patch('charmhelpers.contrib.openstack.context.config') @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' 'resolve_ports') - def test_data_port_name(self, mock_resolve_ports): + def test_data_port_name(self, mock_resolve_ports, config): self.test_config.set('data-port', 'br-data:em1') + config.side_effect = self.test_config.get mock_resolve_ports.side_effect = lambda ports: ports self.assertEquals(context.DataPortContext()(), {'br-data': 'em1'}) - @patch.object(context, 'get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.config') @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @patch('charmhelpers.contrib.openstack.context.list_nics') - def test_data_port_mac(self, list_nics, get_nic_hwaddr, get_nic_hwaddr2): + def test_data_port_mac(self, list_nics, get_nic_hwaddr, config): machine_machs = { 'em1': 'aa:aa:aa:aa:aa:aa', 'eth0': 'bb:bb:bb:bb:bb:bb', } - get_nic_hwaddr2.side_effect = lambda nic: machine_machs[nic] absent_mac = "cc:cc:cc:cc:cc:cc" config_macs = ("br-d1:%s br-d2:%s" % (absent_mac, machine_machs['em1'])) self.test_config.set('data-port', config_macs) + config.side_effect = self.test_config.get list_nics.return_value = machine_machs.keys() get_nic_hwaddr.side_effect = lambda nic: machine_machs[nic] self.assertEquals(context.DataPortContext()(), {'br-d2': 'em1'}) + @patch('charmhelpers.contrib.openstack.context.config') @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' 'resolve_ports') - def test_ensure_bridge_data_port_present(self, mock_resolve_ports): + def test_ensure_bridge_data_port_present(self, mock_resolve_ports, config): self.test_config.set('data-port', 'br-data:em1') self.test_config.set('bridge-mappings', 'phybr1:br-data') + config.side_effect = self.test_config.get def add_port(bridge, port, promisc): + if bridge == 'br-data' and port == 'em1' and promisc is True: self.bridge_added = True return @@ -73,6 +75,9 @@ class OVSPluginContextTest(CharmTestCase): context.OVSPluginContext()._ensure_bridge() self.assertEquals(self.bridge_added, True) + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') @patch.object(charmhelpers.contrib.openstack.context, 'config') @patch.object(charmhelpers.contrib.openstack.context, 'unit_get') @patch.object(charmhelpers.contrib.openstack.context, 'is_clustered') @@ -84,7 +89,7 @@ class OVSPluginContextTest(CharmTestCase): @patch.object(charmhelpers.contrib.openstack.context, 'unit_private_ip') def test_neutroncc_context_api_rel(self, _unit_priv_ip, _npa, _ens_pkgs, _save_ff, _https, _is_clus, _unit_get, - _config): + _config, _runits, _rids, _rget): def mock_npa(plugin, section, manager): if section == "driver": return "neutron.randomdriver" @@ -95,13 +100,15 @@ class OVSPluginContextTest(CharmTestCase): _unit_get.return_value = '127.0.0.13' _unit_priv_ip.return_value = '127.0.0.14' _is_clus.return_value = False - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'l2-population': 'True', - 'network-device-mtu': 1500, - 'overlay-network-type': 'gre', - }) + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'l2-population': 'True', + 'network-device-mtu': 1500, + 'overlay-network-type': 'gre', + } + _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() @@ -128,6 +135,9 @@ class OVSPluginContextTest(CharmTestCase): self.assertEquals(expect, napi_ctxt()) self.service_start.assertCalled() + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') @patch.object(charmhelpers.contrib.openstack.context, 'config') @patch.object(charmhelpers.contrib.openstack.context, 'unit_get') @patch.object(charmhelpers.contrib.openstack.context, 'is_clustered') @@ -142,7 +152,8 @@ class OVSPluginContextTest(CharmTestCase): _ens_pkgs, _save_ff, _https, _is_clus, _unit_get, - _config): + _config, _runits, + _rids, _rget): def mock_npa(plugin, section, manager): if section == "driver": return "neutron.randomdriver" @@ -155,13 +166,15 @@ class OVSPluginContextTest(CharmTestCase): _unit_priv_ip.return_value = '127.0.0.14' _is_clus.return_value = False self.test_config.set('disable-security-groups', True) - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'l2-population': 'True', - 'network-device-mtu': 1500, - 'overlay-network-type': 'gre', - }) + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'l2-population': 'True', + 'network-device-mtu': 1500, + 'overlay-network-type': 'gre', + } + _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() From 0775cd407bf5f89f142c913da2011b602b72bc2f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 14:51:19 +0000 Subject: [PATCH 45/61] Only render network_device_mtu if the option is set --- templates/icehouse/neutron.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index c7af465a..b0e35f9a 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -12,8 +12,9 @@ state_path = /var/lib/neutron lock_path = $state_path/lock bind_host = 0.0.0.0 bind_port = 9696 +{% if network_device_mtu -%} network_device_mtu = {{ network_device_mtu }} - +{% endif -%} {% if core_plugin -%} core_plugin = {{ core_plugin }} {% endif -%} From b2f737313988ec0f4246ee631d4b8c79179c57ed Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Mar 2015 15:06:40 +0000 Subject: [PATCH 46/61] [gnuoy,trivial] Fix unit_tests --- unit_tests/test_neutron_ovs_context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index a7da378b..f01347c4 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -13,6 +13,7 @@ TO_PATCH = [ 'add_bridge_port', 'service_running', 'service_start', + 'service_restart', 'get_host_ip', ] From 1f7afaf55fc40b59bd91cf441dafd74a701f9953 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 24 Mar 2015 17:15:44 +0000 Subject: [PATCH 47/61] Specialize kilo config --- templates/kilo/neutron.conf | 40 ++++++++++++++++++++++++++++++++ templates/parts/section-rabbitmq | 22 ++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 templates/kilo/neutron.conf create mode 100644 templates/parts/section-rabbitmq diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf new file mode 100644 index 00000000..0c209c4e --- /dev/null +++ b/templates/kilo/neutron.conf @@ -0,0 +1,40 @@ +# icehouse +############################################################################### +# [ WARNING ] +# Configuration file maintained by Juju. Local changes may be overwritten. +# Config managed by neutron-openvswitch charm +############################################################################### +[DEFAULT] +verbose = {{ verbose }} +debug = {{ debug }} +use_syslog = {{ use_syslog }} +state_path = /var/lib/neutron +bind_host = 0.0.0.0 +bind_port = 9696 +{% if network_device_mtu -%} +network_device_mtu = {{ network_device_mtu }} +{% endif -%} +{% if core_plugin -%} +core_plugin = {{ core_plugin }} +{% endif -%} + +api_paste_config = /etc/neutron/api-paste.ini +auth_strategy = keystone +notification_driver = neutron.openstack.common.notifier.rpc_notifier +default_notification_level = INFO +notification_topics = notifications + +{% include "parts/section-rabbitmq" %} + +[QUOTAS] + +[DEFAULT_SERVICETYPE] + +[AGENT] +root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf + +[keystone_authtoken] +signing_dir = /var/lib/neutron/keystone-signing + +[oslo_concurrency] +lock_path = $state_path/lock diff --git a/templates/parts/section-rabbitmq b/templates/parts/section-rabbitmq new file mode 100644 index 00000000..df21178b --- /dev/null +++ b/templates/parts/section-rabbitmq @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} \ No newline at end of file From b54c200370509c96d114bdf85531271c226c3262 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 08:37:29 +0000 Subject: [PATCH 48/61] Prep for merging next in --- charm-helpers-sync.yaml | 2 +- hooks/charmhelpers/contrib/network/ip.py | 85 +++++- .../charmhelpers/contrib/openstack/context.py | 267 +++++++++++++++--- .../charmhelpers/contrib/openstack/neutron.py | 70 +++++ hooks/charmhelpers/contrib/openstack/utils.py | 78 +---- hooks/charmhelpers/core/hookenv.py | 26 ++ hooks/charmhelpers/core/host.py | 6 +- hooks/charmhelpers/core/services/helpers.py | 16 +- hooks/neutron_ovs_context.py | 78 +---- hooks/neutron_ovs_utils.py | 4 +- 10 files changed, 433 insertions(+), 199 deletions(-) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 35c175f4..c4fc1cb1 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:~gnuoy/charm-helpers/neutron-contexts +branch: lp:~gnuoy/charm-helpers/neutron-shuffle destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 98b17544..fff6d5ca 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -17,13 +17,16 @@ import glob import re import subprocess +import six +import socket from functools import partial from charmhelpers.core.hookenv import unit_get from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( - log + log, + WARNING, ) try: @@ -365,3 +368,83 @@ def is_bridge_member(nic): return True return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4 address + socket.inet_aton(address) + return True + except socket.error: + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + apt_install('python-dnspython') + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + answers = dns.resolver.query(address, rtype) + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + apt_install("python-dnspython") + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + if not result: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 9385f1a2..45e65790 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -16,17 +16,14 @@ import json import os +import re import time from base64 import b64decode from subprocess import check_call -import re import six +import yaml -from charmhelpers.core.host import ( - list_nics, - get_nic_hwaddr -) from charmhelpers.fetch import ( apt_install, filter_installed_packages, @@ -50,8 +47,11 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string from charmhelpers.core.host import ( + list_nics, + get_nic_hwaddr, mkdir, write_file, ) @@ -68,6 +68,11 @@ from charmhelpers.contrib.hahelpers.apache import ( ) from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, + parse_data_port_mappings, +) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + INTERNAL, ) from charmhelpers.contrib.network.ip import ( get_address_in_network, @@ -79,7 +84,6 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip - CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -111,9 +115,41 @@ def context_complete(ctxt): def config_flags_parser(config_flags): """Parses config flags string into dict. + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + {'key1': 'value1', + 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + {'key1', 'value1', + 'key2', 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + {'key1', 'subkey1=value1, subkey2=value2'} + The provided config_flags string may be a list of comma-separated values which themselves may be comma-separated list of values. """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return yaml.safe_load(config_flags) + if config_flags.find('==') >= 0: log("config_flags is not in expected format (key=value)", level=ERROR) raise OSContextError @@ -198,7 +234,7 @@ class SharedDBContext(OSContextGenerator): unit=local_unit()) if set_hostname != access_hostname: relation_set(relation_settings={hostname_key: access_hostname}) - return ctxt # Defer any further hook execution for now.... + return None # Defer any further hook execution for now.... password_setting = 'password' if self.relation_prefix: @@ -701,7 +737,14 @@ class ApacheSSLContext(OSContextGenerator): 'endpoints': [], 'ext_ports': []} - for cn in self.canonical_names(): + cns = self.canonical_names() + if cns: + for cn in cns: + self.configure_cert(cn) + else: + # Expect cert/key provided in config (currently assumed that ca + # uses ip for cn) + cn = resolve_address(endpoint_type=INTERNAL) self.configure_cert(cn) addresses = self.get_network_addresses() @@ -857,6 +900,48 @@ class NeutronContext(OSContextGenerator): return ctxt +class NeutronPortContext(OSContextGenerator): + NIC_PREFIXES = ['eth', 'bond'] + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + for nic in list_nics(self.NIC_PREFIXES): + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + resolved = [] + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in ports: + if re.match(mac_regex, entry): + # NIC is in known NICs and does NOT hace an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + resolved.append(hwaddr_to_nic[entry]) + else: + # If the passed entry is not a MAC address, assume it's a valid + # interface, and that the user put it there on purpose (we can + # trust it to be the real external network). + resolved.append(entry) + + return resolved + + class OSConfigFlagContext(OSContextGenerator): """Provides support for user-defined config flags. @@ -1080,53 +1165,143 @@ class SysctlContext(OSContextGenerator): return {'sysctl': sysctl_dict} -class NeutronPortContext(OSContextGenerator): +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] - def _resolve_port(self, config_key): - if not config(config_key): - return None - hwaddr_to_nic = {} - hwaddr_to_ip = {} - for nic in list_nics(['eth', 'bond']): - hwaddr = get_nic_hwaddr(nic) - hwaddr_to_nic[hwaddr] = nic - addresses = get_ipv4_addr(nic, fatal=False) + \ - get_ipv6_addr(iface=nic, fatal=False) - hwaddr_to_ip[hwaddr] = addresses - mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) - for entry in config(config_key).split(): - entry = entry.strip() - if re.match(mac_regex, entry): - if entry in hwaddr_to_nic and len(hwaddr_to_ip[entry]) == 0: - # If the nic is part of a bridge then don't use it - if is_bridge_member(hwaddr_to_nic[entry]): - continue - # Entry is a MAC address for a valid interface that doesn't - # have an IP address assigned yet. - return hwaddr_to_nic[entry] + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] else: - # If the passed entry is not a MAC address, assume it's a valid - # interface, and that the user put it there on purpose (we can - # trust it to be the real external network). - return entry - return None + settings[nkey] = defv + return settings class ExternalPortContext(NeutronPortContext): def __call__(self): - port = self._resolve_port('ext-port') - if port: - return {"ext_port": port} - else: - return None + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt class DataPortContext(NeutronPortContext): def __call__(self): - port = self._resolve_port('data-port') - if port: - return {"data_port": port} - else: - return None + ports = config('data-port') + if ports: + portmap = parse_data_port_mappings(ports) + ports = portmap.values() + resolved = self.resolve_ports(ports) + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {bridge: normalized[port] for bridge, port in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.values(): + ports = mappings.values() + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt["devs"] = '\\n'.join(ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + } + if context_complete(ctxt): + return ctxt + return {} diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py index 902757fe..f8851050 100644 --- a/hooks/charmhelpers/contrib/openstack/neutron.py +++ b/hooks/charmhelpers/contrib/openstack/neutron.py @@ -16,6 +16,7 @@ # Various utilies for dealing with Neutron and the renaming from Quantum. +import six from subprocess import check_output from charmhelpers.core.hookenv import ( @@ -237,3 +238,72 @@ def network_manager(): else: # ensure accurate naming for all releases post-H return 'neutron' + + +def parse_mappings(mappings): + parsed = {} + if mappings: + mappings = mappings.split(' ') + for m in mappings: + p = m.partition(':') + if p[1] == ':': + parsed[p[0].strip()] = p[2].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings, default_bridge='br-data'): + """Parse data port mappings. + + Mappings must be a space-delimited list of bridge:port mappings. + + Returns dict of the form {bridge:port}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {default_bridge: mappings.split(' ')[0]} + + bridges = _mappings.keys() + ports = _mappings.values() + if len(set(bridges)) != len(bridges): + raise Exception("It is not allowed to have more than one port " + "configured on the same bridge") + + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return _mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + return {} + + mappings = {} + for p, r in six.iteritems(_mappings): + mappings[p] = tuple(r.split(':')) + + return mappings diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index af2b3596..4f110c63 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -23,12 +23,13 @@ from functools import wraps import subprocess import json import os -import socket import sys import six import yaml +from charmhelpers.contrib.network import ip + from charmhelpers.core.hookenv import ( config, log as juju_log, @@ -421,77 +422,10 @@ def clean_storage(block_device): else: zap_disk(block_device) - -def is_ip(address): - """ - Returns True if address is a valid IP address. - """ - try: - # Test to see if already an IPv4 address - socket.inet_aton(address) - return True - except socket.error: - return False - - -def ns_query(address): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython') - import dns.resolver - - if isinstance(address, dns.name.Name): - rtype = 'PTR' - elif isinstance(address, six.string_types): - rtype = 'A' - else: - return None - - answers = dns.resolver.query(address, rtype) - if answers: - return str(answers[0]) - return None - - -def get_host_ip(hostname): - """ - Resolves the IP for a given hostname, or returns - the input if it is already an IP. - """ - if is_ip(hostname): - return hostname - - return ns_query(hostname) - - -def get_hostname(address, fqdn=True): - """ - Resolves hostname for given IP, or returns the input - if it is already a hostname. - """ - if is_ip(address): - try: - import dns.reversename - except ImportError: - apt_install('python-dnspython') - import dns.reversename - - rev = dns.reversename.from_address(address) - result = ns_query(rev) - if not result: - return None - else: - result = address - - if fqdn: - # strip trailing . - if result.endswith('.'): - return result[:-1] - else: - return result - else: - return result.split('.')[0] +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index cf552b39..715dd4c5 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -566,3 +566,29 @@ class Hooks(object): def charm_dir(): """Return the root directory of the current charm""" return os.environ.get('CHARM_DIR') + + +@cached +def action_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +def action_set(values): + """Sets the values to be returned after the action finishes""" + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def action_fail(message): + """Sets the action status to failed and sets the error message. + + The results set by action_set are preserved.""" + subprocess.check_call(['action-fail', message]) diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index b771c611..830822af 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -339,12 +339,16 @@ def lsb_release(): def pwgen(length=None): """Generate a random pasword.""" if length is None: + # A random length is ok to use a weak PRNG length = random.choice(range(35, 45)) alphanumeric_chars = [ l for l in (string.ascii_letters + string.digits) if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() random_chars = [ - random.choice(alphanumeric_chars) for _ in range(length)] + random_generator.choice(alphanumeric_chars) for _ in range(length)] return(''.join(random_chars)) diff --git a/hooks/charmhelpers/core/services/helpers.py b/hooks/charmhelpers/core/services/helpers.py index 5e3af9da..3eb5fb44 100644 --- a/hooks/charmhelpers/core/services/helpers.py +++ b/hooks/charmhelpers/core/services/helpers.py @@ -45,12 +45,14 @@ class RelationContext(dict): """ name = None interface = None - required_keys = [] def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + if name is not None: self.name = name - if additional_required_keys is not None: + if additional_required_keys: self.required_keys.extend(additional_required_keys) self.get_data() @@ -134,7 +136,10 @@ class MysqlRelation(RelationContext): """ name = 'db' interface = 'mysql' - required_keys = ['host', 'user', 'password', 'database'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + RelationContext.__init__(self, *args, **kwargs) class HttpRelation(RelationContext): @@ -146,7 +151,10 @@ class HttpRelation(RelationContext): """ name = 'website' interface = 'http' - required_keys = ['host', 'port'] + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + RelationContext.__init__(self, *args, **kwargs) def provide_data(self): return { diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 62bec11a..513c577b 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -1,9 +1,6 @@ import os import uuid from charmhelpers.core.hookenv import ( - relation_ids, - related_units, - relation_get, config, unit_get, ) @@ -12,14 +9,13 @@ from charmhelpers.contrib.network.ip import ( ) from charmhelpers.contrib.openstack.ip import resolve_address from charmhelpers.core.host import list_nics, get_nic_hwaddr -from charmhelpers.core.strutils import bool_from_string from charmhelpers.contrib.openstack import context from charmhelpers.core.host import service_running, service_start from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.contrib.openstack.context import ( OSContextGenerator, - context_complete, + NeutronAPIContext, ) import re @@ -28,47 +24,6 @@ OVS_BRIDGE = 'br-int' DATA_BRIDGE = 'br-data' -def _neutron_api_settings(): - ''' - Inspects current neutron-plugin relation - ''' - neutron_settings = { - 'neutron_security_groups': False, - 'l2_population': True, - 'overlay_network_type': 'gre', - 'enable_dvr': False, - } - for rid in relation_ids('neutron-plugin-api'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if 'l2-population' not in rdata: - continue - neutron_settings = { - 'l2_population': bool_from_string(rdata['l2-population']), - 'overlay_network_type': rdata['overlay-network-type'], - 'neutron_security_groups': bool_from_string( - rdata['neutron-security-groups'] - ), - } - if 'enable-dvr' in rdata: - neutron_settings['enable_dvr'] = bool_from_string( - rdata['enable-dvr'] - ) - # Override with configuration if set to true - if config('disable-security-groups'): - neutron_settings['neutron_security_groups'] = False - return neutron_settings - return neutron_settings - - -def use_dvr(): - api_settings = _neutron_api_settings() - if 'enable_dvr' in api_settings: - return api_settings['enable_dvr'] - else: - return False - - class OVSPluginContext(context.NeutronContext): interfaces = [] @@ -82,7 +37,7 @@ class OVSPluginContext(context.NeutronContext): @property def neutron_security_groups(self): - neutron_api_settings = _neutron_api_settings() + neutron_api_settings = NeutronAPIContext()() return neutron_api_settings['neutron_security_groups'] def get_data_port(self): @@ -125,10 +80,10 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['local_ip'] = \ get_address_in_network(config('os-data-network'), get_host_ip(unit_get('private-address'))) - neutron_api_settings = _neutron_api_settings() + neutron_api_settings = NeutronAPIContext()() ovs_ctxt['neutron_security_groups'] = self.neutron_security_groups ovs_ctxt['l2_population'] = neutron_api_settings['l2_population'] - ovs_ctxt['distributed_routing'] = use_dvr() + ovs_ctxt['distributed_routing'] = neutron_api_settings['enable_dvr'] ovs_ctxt['overlay_network_type'] = \ neutron_api_settings['overlay_network_type'] # TODO: We need to sort out the syslog and debug/verbose options as a @@ -142,7 +97,7 @@ class OVSPluginContext(context.NeutronContext): class L3AgentContext(OSContextGenerator): def __call__(self): - neutron_api_settings = _neutron_api_settings() + neutron_api_settings = NeutronAPIContext()() ctxt = {} if neutron_api_settings['enable_dvr']: ctxt['agent_mode'] = 'dvr' @@ -151,27 +106,6 @@ class L3AgentContext(OSContextGenerator): return ctxt -class NetworkServiceContext(OSContextGenerator): - interfaces = ['neutron-network-service'] - - def __call__(self): - for rid in relation_ids('neutron-network-service'): - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - ctxt = { - 'service_protocol': - rdata.get('service_protocol') or 'http', - 'keystone_host': rdata.get('keystone_host'), - 'service_port': rdata.get('service_port'), - 'region': rdata.get('region'), - 'service_tenant': rdata.get('service_tenant'), - 'service_username': rdata.get('service_username'), - 'service_password': rdata.get('service_password'), - } - if context_complete(ctxt): - return ctxt - - SHARED_SECRET = "/etc/neutron/secret.txt" @@ -190,7 +124,7 @@ def get_shared_secret(): class DVRSharedSecretContext(OSContextGenerator): def __call__(self): - if use_dvr(): + if NeutronAPIContext()()['enable_dvr']: ctxt = { 'shared_secret': get_shared_secret(), 'local_ip': resolve_address(), diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 252656a0..7ad866a3 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -50,7 +50,7 @@ DVR_RESOURCE_MAP = OrderedDict([ (NEUTRON_METADATA_AGENT_CONF, { 'services': ['neutron-metadata-agent'], 'contexts': [neutron_ovs_context.DVRSharedSecretContext(), - neutron_ovs_context.NetworkServiceContext()], + context.NetworkServiceContext()], }), ]) TEMPLATES = 'templates/' @@ -118,4 +118,4 @@ def get_shared_secret(): def use_dvr(): - return neutron_ovs_context.use_dvr() + return context.NeutronAPIContext()()['enable_dvr'] From 44e8877df863dd972552ddb6b073f290c02e4cf0 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Mar 2015 09:18:34 +0000 Subject: [PATCH 49/61] resync helpers --- .../templates/section-keystone-authtoken | 9 ++++++++ .../openstack/templates/section-rabbitmq-oslo | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken create mode 100644 hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken new file mode 100644 index 00000000..2a37edd5 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -0,0 +1,9 @@ +{% if auth_host -%} +[keystone_authtoken] +identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo new file mode 100644 index 00000000..b444c9c9 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} From 68a33867f02da0e532649831f15680d798322eeb Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 25 Mar 2015 09:22:37 +0000 Subject: [PATCH 50/61] Switch to using charm-helper templates for rabbitmq --- templates/kilo/neutron.conf | 2 +- templates/parts/section-rabbitmq | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 templates/parts/section-rabbitmq diff --git a/templates/kilo/neutron.conf b/templates/kilo/neutron.conf index 0c209c4e..b4703bef 100644 --- a/templates/kilo/neutron.conf +++ b/templates/kilo/neutron.conf @@ -24,7 +24,7 @@ notification_driver = neutron.openstack.common.notifier.rpc_notifier default_notification_level = INFO notification_topics = notifications -{% include "parts/section-rabbitmq" %} +{% include "section-rabbitmq-oslo" %} [QUOTAS] diff --git a/templates/parts/section-rabbitmq b/templates/parts/section-rabbitmq deleted file mode 100644 index df21178b..00000000 --- a/templates/parts/section-rabbitmq +++ /dev/null @@ -1,22 +0,0 @@ -{% if rabbitmq_host or rabbitmq_hosts -%} -[oslo_messaging_rabbit] -rabbit_userid = {{ rabbitmq_user }} -rabbit_virtual_host = {{ rabbitmq_virtual_host }} -rabbit_password = {{ rabbitmq_password }} -{% if rabbitmq_hosts -%} -rabbit_hosts = {{ rabbitmq_hosts }} -{% if rabbitmq_ha_queues -%} -rabbit_ha_queues = True -rabbit_durable_queues = False -{% endif -%} -{% else -%} -rabbit_host = {{ rabbitmq_host }} -{% endif -%} -{% if rabbit_ssl_port -%} -rabbit_use_ssl = True -rabbit_port = {{ rabbit_ssl_port }} -{% if rabbit_ssl_ca -%} -kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} -{% endif -%} -{% endif -%} -{% endif -%} \ No newline at end of file From 7f32b61062464c8c29fbcfeb36a3cd08ee5848c8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 10:29:27 +0000 Subject: [PATCH 51/61] Fixed empty metadata info --- hooks/neutron_ovs_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index ee7435bd..8cdcb3c4 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -56,7 +56,9 @@ DVR_RESOURCE_MAP = OrderedDict([ (NEUTRON_METADATA_AGENT_CONF, { 'services': ['neutron-metadata-agent'], 'contexts': [neutron_ovs_context.DVRSharedSecretContext(), - context.NetworkServiceContext()], + context.NetworkServiceContext( + rel_name='neutron-network-service' + )], }), ]) TEMPLATES = 'templates/' From 0f1309df41afff67ae4c66c73bd6dfa8c5982633 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Mar 2015 10:31:44 +0000 Subject: [PATCH 52/61] Fixed lint --- unit_tests/test_neutron_ovs_context.py | 9 ++++++--- unit_tests/test_neutron_ovs_hooks.py | 1 - unit_tests/test_neutron_ovs_utils.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index c0602396..0232aecb 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -16,6 +16,7 @@ TO_PATCH = [ 'get_host_ip', ] + def fake_context(settings): def outer(): def inner(): @@ -23,6 +24,7 @@ def fake_context(settings): return inner return outer + class OVSPluginContextTest(CharmTestCase): def setUp(self): @@ -262,7 +264,6 @@ class DVRSharedSecretContext(CharmTestCase): TO_PATCH) self.config.side_effect = self.test_config.get - @patch('os.path') @patch('uuid.uuid4') def test_secret_created_stored(self, _uuid4, _path): @@ -287,7 +288,8 @@ class DVRSharedSecretContext(CharmTestCase): @patch.object(context, 'NeutronAPIContext') @patch.object(context, 'get_shared_secret') - def test_shared_secretcontext_dvr(self, _shared_secret, _NeutronAPIContext): + def test_shared_secretcontext_dvr(self, _shared_secret, + _NeutronAPIContext): _NeutronAPIContext.side_effect = fake_context({'enable_dvr': True}) _shared_secret.return_value = 'secret_thing' #_use_dvr.return_value = True @@ -298,7 +300,8 @@ class DVRSharedSecretContext(CharmTestCase): @patch.object(context, 'NeutronAPIContext') @patch.object(context, 'get_shared_secret') - def test_shared_secretcontext_nodvr(self, _shared_secret, _NeutronAPIContext): + def test_shared_secretcontext_nodvr(self, _shared_secret, + _NeutronAPIContext): _NeutronAPIContext.side_effect = fake_context({'enable_dvr': False}) _shared_secret.return_value = 'secret_thing' self.resolve_address.return_value = '10.0.0.10' diff --git a/unit_tests/test_neutron_ovs_hooks.py b/unit_tests/test_neutron_ovs_hooks.py index b167c886..d651d00f 100644 --- a/unit_tests/test_neutron_ovs_hooks.py +++ b/unit_tests/test_neutron_ovs_hooks.py @@ -1,7 +1,6 @@ from mock import MagicMock, patch, call from test_utils import CharmTestCase -import neutron_ovs_context with patch('charmhelpers.core.hookenv.config') as config: config.return_value = 'neutron' diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 3ad82284..7532d21d 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -61,7 +61,7 @@ class TestNeutronOVSUtils(CharmTestCase): # Reset cached cache hookenv.cache = {} - @patch.object(nutils, 'use_dvr') + @patch.object(nutils, 'use_dvr') @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release') @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package') def test_determine_packages(self, _head_pkgs, _os_rel, _use_dvr): @@ -72,7 +72,7 @@ class TestNeutronOVSUtils(CharmTestCase): expect = [['neutron-plugin-openvswitch-agent'], [head_pkg]] self.assertItemsEqual(pkg_list, expect) - @patch.object(nutils, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_register_configs(self, _use_dvr): class _mock_OSConfigRenderer(): def __init__(self, templates_dir=None, openstack_release=None): @@ -92,7 +92,7 @@ class TestNeutronOVSUtils(CharmTestCase): '/etc/init/os-charm-phy-nic-mtu.conf'] self.assertItemsEqual(_regconfs.configs, confs) - @patch.object(nutils, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_resource_map(self, _use_dvr): _use_dvr.return_value = False _map = nutils.resource_map() @@ -101,7 +101,7 @@ class TestNeutronOVSUtils(CharmTestCase): [self.assertIn(q_conf, _map.keys()) for q_conf in confs] self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) - @patch.object(nutils, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_resource_map_dvr(self, _use_dvr): _use_dvr.return_value = True _map = nutils.resource_map() @@ -111,7 +111,7 @@ class TestNeutronOVSUtils(CharmTestCase): [self.assertIn(q_conf, _map.keys()) for q_conf in confs] self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) - @patch.object(nutils, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_restart_map(self, _use_dvr): _use_dvr.return_value = False _restart_map = nutils.restart_map() From f87b3d5a6d3b595258b0658bd55541550a10b931 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Mar 2015 10:00:52 +0000 Subject: [PATCH 53/61] Add mtu fixes to configure ovs and fix unit tests --- hooks/neutron_ovs_utils.py | 34 ++++++++++++- unit_tests/test_neutron_ovs_utils.py | 73 ++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 6 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 8cdcb3c4..e82c28c3 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -10,6 +10,21 @@ import neutron_ovs_context from charmhelpers.contrib.network.ovs import ( add_bridge, add_bridge_port, + full_restart, +) +from charmhelpers.core.hookenv import ( + config, +) +from charmhelpers.contrib.openstack.neutron import ( + parse_bridge_mappings, +) +from charmhelpers.contrib.openstack.context import ( + ExternalPortContext, + DataPortContext, +) +from charmhelpers.core.host import ( + service_restart, + service_running, ) NOVA_CONF_DIR = "/etc/nova" @@ -110,13 +125,28 @@ def restart_map(): def configure_ovs(): + if not service_running('openvswitch-switch'): + full_restart() add_bridge(INT_BRIDGE) add_bridge(EXT_BRIDGE) - ext_port_ctx = context.ExternalPortContext()() + ext_port_ctx = ExternalPortContext()() if ext_port_ctx and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) - add_bridge(DATA_BRIDGE) + portmaps = DataPortContext()() + bridgemaps = parse_bridge_mappings(config('bridge-mappings')) + print bridgemaps + for provider, br in bridgemaps.iteritems(): + add_bridge(br) + print portmaps + if not portmaps or br not in portmaps: + continue + + add_bridge_port(br, portmaps[br], promisc=True) + + # Ensure this runs so that mtu is applied to data-port interfaces if + # provided. + service_restart('os-charm-phy-nic-mtu') def get_shared_secret(): diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 7532d21d..f0e6bb6d 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -2,7 +2,6 @@ from mock import MagicMock, patch, call from collections import OrderedDict import charmhelpers.contrib.openstack.templating as templating -from charmhelpers.contrib.openstack import context templating.OSConfigRenderer = MagicMock() @@ -19,8 +18,13 @@ import charmhelpers.core.hookenv as hookenv TO_PATCH = [ 'add_bridge', 'add_bridge_port', + 'config', 'os_release', 'neutron_plugin_attribute', + 'full_restart', + 'service_running', + 'service_restart', + 'ExternalPortContext', ] head_pkg = 'linux-headers-3.15.0-5-generic' @@ -56,6 +60,7 @@ class TestNeutronOVSUtils(CharmTestCase): def setUp(self): super(TestNeutronOVSUtils, self).setUp(nutils, TO_PATCH) self.neutron_plugin_attribute.side_effect = _mock_npa + self.config.side_effect = self.test_config.get def tearDown(self): # Reset cached cache @@ -126,9 +131,58 @@ class TestNeutronOVSUtils(CharmTestCase): self.assertTrue(item in _restart_map) self.assertTrue(expect[item] == _restart_map[item]) - @patch.object(context, 'ExternalPortContext') - def test_configure_ovs_ovs_ext_port(self, _ext_port_ctxt): - _ext_port_ctxt.return_value = \ + @patch('charmhelpers.contrib.openstack.context.config') + def test_configure_ovs_ovs_data_port(self, mock_config): + mock_config.side_effect = self.test_config.get + self.config.side_effect = self.test_config.get + self.ExternalPortContext.return_value = \ + DummyContext(return_value=None) + # Test back-compatibility i.e. port but no bridge (so br-data is + # assumed) + self.test_config.set('data-port', 'eth0') + nutils.configure_ovs() + self.add_bridge.assert_has_calls([ + call('br-int'), + call('br-ex'), + call('br-data') + ]) + self.assertTrue(self.add_bridge_port.called) + + # Now test with bridge:port format + self.test_config.set('data-port', 'br-foo:eth0') + self.add_bridge.reset_mock() + self.add_bridge_port.reset_mock() + nutils.configure_ovs() + self.add_bridge.assert_has_calls([ + call('br-int'), + call('br-ex'), + call('br-data') + ]) + # Not called since we have a bogus bridge in data-ports + self.assertFalse(self.add_bridge_port.called) + + @patch('charmhelpers.contrib.openstack.context.config') + def test_configure_ovs_starts_service_if_required(self, mock_config): + mock_config.side_effect = self.test_config.get + self.config.return_value = 'ovs' + self.service_running.return_value = False + nutils.configure_ovs() + self.assertTrue(self.full_restart.called) + + @patch('charmhelpers.contrib.openstack.context.config') + def test_configure_ovs_doesnt_restart_service(self, mock_config): + mock_config.side_effect = self.test_config.get + self.config.side_effect = self.test_config.get + self.service_running.return_value = True + nutils.configure_ovs() + self.assertFalse(self.full_restart.called) + + @patch('charmhelpers.contrib.openstack.context.config') + def test_configure_ovs_ovs_ext_port(self, mock_config): + mock_config.side_effect = self.test_config.get + self.config.side_effect = self.test_config.get + self.test_config.set('ext-port', 'eth0') + self.ExternalPortContext.return_value = \ DummyContext(return_value={'ext_port': 'eth0'}) nutils.configure_ovs() self.add_bridge.assert_has_calls([ @@ -137,6 +191,17 @@ class TestNeutronOVSUtils(CharmTestCase): call('br-data') ]) self.add_bridge_port.assert_called_with('br-ex', 'eth0') +# @patch.object(context, 'ExternalPortContext') +# def test_configure_ovs_ovs_ext_port(self, _ext_port_ctxt): +# _ext_port_ctxt.return_value = \ +# DummyContext(return_value={'ext_port': 'eth0'}) +# nutils.configure_ovs() +# self.add_bridge.assert_has_calls([ +# call('br-int'), +# call('br-ex'), +# call('br-data') +# ]) +# self.add_bridge_port.assert_called_with('br-ex', 'eth0') @patch.object(neutron_ovs_context, 'DVRSharedSecretContext') def test_get_shared_secret(self, _dvr_secret_ctxt): From c13cd7e0c8d69da205b8a35675687e0c5f778ea1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Mar 2015 10:22:41 +0000 Subject: [PATCH 54/61] Stop managing the bridges in two places --- hooks/neutron_ovs_context.py | 19 ------------------- hooks/neutron_ovs_utils.py | 6 +++--- unit_tests/test_neutron_ovs_context.py | 20 -------------------- unit_tests/test_neutron_ovs_utils.py | 17 +++++++++++++---- 4 files changed, 16 insertions(+), 46 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index a7192a4e..c5d4b1d1 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -44,23 +44,6 @@ class OVSPluginContext(context.NeutronContext): neutron_api_settings = NeutronAPIContext()() return neutron_api_settings['neutron_security_groups'] - def _ensure_bridge(self): - if not service_running('openvswitch-switch'): - service_start('openvswitch-switch') - - add_bridge(OVS_BRIDGE) - - portmaps = DataPortContext()() - bridgemaps = parse_bridge_mappings(config('bridge-mappings')) - for provider, br in bridgemaps.iteritems(): - add_bridge(br) - - if not portmaps or br not in portmaps: - continue - - add_bridge_port(br, portmaps[br], promisc=True) - - service_restart('os-charm-phy-nic-mtu') def ovs_ctxt(self): # In addition to generating config context, ensure the OVS service @@ -70,8 +53,6 @@ class OVSPluginContext(context.NeutronContext): if not ovs_ctxt: return {} - self._ensure_bridge() - conf = config() ovs_ctxt['local_ip'] = \ get_address_in_network(config('os-data-network'), diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index e82c28c3..fccf9309 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -129,16 +129,16 @@ def configure_ovs(): full_restart() add_bridge(INT_BRIDGE) add_bridge(EXT_BRIDGE) - ext_port_ctx = ExternalPortContext()() + ext_port_ctx = None + if use_dvr(): + ext_port_ctx = ExternalPortContext()() if ext_port_ctx and ext_port_ctx['ext_port']: add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port']) portmaps = DataPortContext()() bridgemaps = parse_bridge_mappings(config('bridge-mappings')) - print bridgemaps for provider, br in bridgemaps.iteritems(): add_bridge(br) - print portmaps if not portmaps or br not in portmaps: continue diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 0232aecb..93fab0ff 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -65,26 +65,6 @@ class OVSPluginContextTest(CharmTestCase): self.assertEquals(context.DataPortContext()(), {'br-d2': 'em1'}) - @patch('charmhelpers.contrib.openstack.context.config') - @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' - 'resolve_ports') - def test_ensure_bridge_data_port_present(self, mock_resolve_ports, config): - self.test_config.set('data-port', 'br-data:em1') - self.test_config.set('bridge-mappings', 'phybr1:br-data') - config.side_effect = self.test_config.get - - def add_port(bridge, port, promisc): - - if bridge == 'br-data' and port == 'em1' and promisc is True: - self.bridge_added = True - return - self.bridge_added = False - - mock_resolve_ports.side_effect = lambda ports: ports - self.add_bridge_port.side_effect = add_port - context.OVSPluginContext()._ensure_bridge() - self.assertEquals(self.bridge_added, True) - @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') @patch.object(charmhelpers.contrib.openstack.context, 'related_units') diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index f0e6bb6d..7eb798f7 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -131,8 +131,10 @@ class TestNeutronOVSUtils(CharmTestCase): self.assertTrue(item in _restart_map) self.assertTrue(expect[item] == _restart_map[item]) + @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_ovs_data_port(self, mock_config): + def test_configure_ovs_ovs_data_port(self, mock_config, _use_dvr): + _use_dvr.return_value = False mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.ExternalPortContext.return_value = \ @@ -161,24 +163,31 @@ class TestNeutronOVSUtils(CharmTestCase): # Not called since we have a bogus bridge in data-ports self.assertFalse(self.add_bridge_port.called) + @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_starts_service_if_required(self, mock_config): + def test_configure_ovs_starts_service_if_required(self, mock_config, + _use_dvr): + _use_dvr = False mock_config.side_effect = self.test_config.get self.config.return_value = 'ovs' self.service_running.return_value = False nutils.configure_ovs() self.assertTrue(self.full_restart.called) + @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_doesnt_restart_service(self, mock_config): + def test_configure_ovs_doesnt_restart_service(self, mock_config, _usedvr): + _use_dvr = False mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.service_running.return_value = True nutils.configure_ovs() self.assertFalse(self.full_restart.called) + @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_ovs_ext_port(self, mock_config): + def test_configure_ovs_ovs_ext_port(self, mock_config, _usedvr): + _use_dvr = False mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.test_config.set('ext-port', 'eth0') From 8f1827f88c7a1fadd8e383f0c2501dfb64503500 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Mar 2015 10:27:32 +0000 Subject: [PATCH 55/61] Lint cleanup --- hooks/neutron_ovs_context.py | 10 ---------- unit_tests/test_neutron_ovs_utils.py | 21 +++++---------------- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index c5d4b1d1..e6b71a4c 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -6,24 +6,15 @@ from charmhelpers.core.hookenv import ( ) from charmhelpers.contrib.openstack.ip import resolve_address from charmhelpers.contrib.openstack import context -from charmhelpers.core.host import ( - service_running, - service_start, - service_restart, -) -from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip from charmhelpers.contrib.network.ip import get_address_in_network from charmhelpers.contrib.openstack.context import ( OSContextGenerator, NeutronAPIContext, - DataPortContext, ) from charmhelpers.contrib.openstack.neutron import ( - parse_bridge_mappings, parse_vlan_range_mappings, ) -OVS_BRIDGE = 'br-int' class OVSPluginContext(context.NeutronContext): @@ -44,7 +35,6 @@ class OVSPluginContext(context.NeutronContext): neutron_api_settings = NeutronAPIContext()() return neutron_api_settings['neutron_security_groups'] - def ovs_ctxt(self): # In addition to generating config context, ensure the OVS service # is running and the OVS bridge exists. Also need to ensure diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 7eb798f7..abe5a3a5 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -167,7 +167,7 @@ class TestNeutronOVSUtils(CharmTestCase): @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_starts_service_if_required(self, mock_config, _use_dvr): - _use_dvr = False + _use_dvr.return_value = False mock_config.side_effect = self.test_config.get self.config.return_value = 'ovs' self.service_running.return_value = False @@ -176,8 +176,8 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_doesnt_restart_service(self, mock_config, _usedvr): - _use_dvr = False + def test_configure_ovs_doesnt_restart_service(self, mock_config, _use_dvr): + _use_dvr.return_value = False mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.service_running.return_value = True @@ -186,8 +186,8 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') - def test_configure_ovs_ovs_ext_port(self, mock_config, _usedvr): - _use_dvr = False + def test_configure_ovs_ovs_ext_port(self, mock_config, _use_dvr): + _use_dvr.return_value = False mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.test_config.set('ext-port', 'eth0') @@ -200,17 +200,6 @@ class TestNeutronOVSUtils(CharmTestCase): call('br-data') ]) self.add_bridge_port.assert_called_with('br-ex', 'eth0') -# @patch.object(context, 'ExternalPortContext') -# def test_configure_ovs_ovs_ext_port(self, _ext_port_ctxt): -# _ext_port_ctxt.return_value = \ -# DummyContext(return_value={'ext_port': 'eth0'}) -# nutils.configure_ovs() -# self.add_bridge.assert_has_calls([ -# call('br-int'), -# call('br-ex'), -# call('br-data') -# ]) -# self.add_bridge_port.assert_called_with('br-ex', 'eth0') @patch.object(neutron_ovs_context, 'DVRSharedSecretContext') def test_get_shared_secret(self, _dvr_secret_ctxt): From a429ae1a8b004543762b3bfe28499861f3475291 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Mar 2015 10:35:54 +0000 Subject: [PATCH 56/61] Fix juno templates --- charm-helpers-sync.yaml | 2 +- templates/juno/ml2_conf.ini | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index c4fc1cb1..9b5e79e9 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:~gnuoy/charm-helpers/neutron-shuffle +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/templates/juno/ml2_conf.ini b/templates/juno/ml2_conf.ini index 1a0c7c93..7cf2706a 100644 --- a/templates/juno/ml2_conf.ini +++ b/templates/juno/ml2_conf.ini @@ -16,20 +16,23 @@ tunnel_id_ranges = 1:1000 vni_ranges = 1001:2000 [ml2_type_vlan] -network_vlan_ranges = physnet1:1000:2000 +network_vlan_ranges = {{ vlan_ranges }} [ml2_type_flat] -flat_networks = physnet1 +flat_networks = {{ network_providers }} [ovs] enable_tunneling = True local_ip = {{ local_ip }} -bridge_mappings = physnet1:br-data +bridge_mappings = {{ bridge_mappings }} [agent] tunnel_types = {{ overlay_network_type }} l2_population = {{ l2_population }} enable_distributed_routing = {{ distributed_routing }} +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} [securitygroup] {% if neutron_security_groups -%} From 502156897b283bc03eb2a9783da01d92e5824e16 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 30 Mar 2015 15:10:41 +0000 Subject: [PATCH 57/61] Update with fixes from mp --- hooks/neutron_ovs_utils.py | 10 +++++----- templates/ext-port.conf | 11 +++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index fccf9309..1c892304 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -36,7 +36,7 @@ NEUTRON_FWAAS_CONF = "/etc/neutron/fwaas_driver.ini" ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR EXT_PORT_CONF = '/etc/init/ext-port.conf' NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini" -DVR_PACKAGES = ['neutron-vpn-agent'] +DVR_PACKAGES = ['neutron-l3-agent'] PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf' TEMPLATES = 'templates/' @@ -57,15 +57,15 @@ BASE_RESOURCE_MAP = OrderedDict([ ]) DVR_RESOURCE_MAP = OrderedDict([ (NEUTRON_L3_AGENT_CONF, { - 'services': ['neutron-vpn-agent'], + 'services': ['neutron-l3-agent'], 'contexts': [neutron_ovs_context.L3AgentContext()], }), (NEUTRON_FWAAS_CONF, { - 'services': ['neutron-vpn-agent'], + 'services': ['neutron-l3-agent'], 'contexts': [neutron_ovs_context.L3AgentContext()], }), (EXT_PORT_CONF, { - 'services': ['neutron-vpn-agent'], + 'services': ['neutron-l3-agent'], 'contexts': [context.ExternalPortContext()], }), (NEUTRON_METADATA_AGENT_CONF, { @@ -111,7 +111,7 @@ def resource_map(): resource_map = deepcopy(BASE_RESOURCE_MAP) if use_dvr(): resource_map.update(DVR_RESOURCE_MAP) - dvr_services = ['neutron-metadata-agent', 'neutron-vpn-agent'] + dvr_services = ['neutron-metadata-agent', 'neutron-l3-agent'] resource_map[NEUTRON_CONF]['services'] += dvr_services return resource_map diff --git a/templates/ext-port.conf b/templates/ext-port.conf index 6080c30e..1d850240 100644 --- a/templates/ext-port.conf +++ b/templates/ext-port.conf @@ -5,5 +5,12 @@ start on runlevel [2345] task script - ip link set {{ ext_port }} up -end script \ No newline at end of file + EXT_PORT="{{ ext_port }}" + MTU="{{ ext_port_mtu }}" + if [ -n "$EXT_PORT" ]; then + ip link set $EXT_PORT up + if [ -n "$MTU" ]; then + ip link set $EXT_PORT mtu $MTU + fi + fi +end script From fe1406ebeceb27af52bb7be2242a63452127876b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 31 Mar 2015 14:33:06 +0000 Subject: [PATCH 58/61] Get keystone creds from neutron-api not nova-cc --- .../charmhelpers/contrib/openstack/context.py | 9 ++++---- hooks/neutron_ovs_context.py | 22 +++++++++++++++++++ hooks/neutron_ovs_utils.py | 4 +--- metadata.yaml | 2 -- templates/juno/metadata_agent.ini | 9 ++++---- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 45e65790..dd51bfbb 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -320,14 +320,15 @@ def db_ssl(rdata, ctxt, ssl_dir): class IdentityServiceContext(OSContextGenerator): - interfaces = ['identity-service'] - def __init__(self, service=None, service_user=None): + def __init__(self, service=None, service_user=None, rel_name='identity-service'): self.service = service self.service_user = service_user + self.rel_name = rel_name + self.interfaces = [self.rel_name] def __call__(self): - log('Generating template context for identity-service', level=DEBUG) + log('Generating template context for ' + self.rel_name, level=DEBUG) ctxt = {} if self.service and self.service_user: @@ -341,7 +342,7 @@ class IdentityServiceContext(OSContextGenerator): ctxt['signing_dir'] = cachedir - for rid in relation_ids('identity-service'): + for rid in relation_ids(self.rel_name): for unit in related_units(rid): rdata = relation_get(rid=rid, unit=unit) serv_host = rdata.get('service_host') diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index e6b71a4c..be1a7839 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -2,6 +2,9 @@ import os import uuid from charmhelpers.core.hookenv import ( config, + relation_get, + relation_ids, + related_units, unit_get, ) from charmhelpers.contrib.openstack.ip import resolve_address @@ -118,3 +121,22 @@ class DVRSharedSecretContext(OSContextGenerator): else: ctxt = {} return ctxt + +class APIIdentityServiceContext(context.IdentityServiceContext): + + def __init__(self): + super(APIIdentityServiceContext, + self).__init__(rel_name='neutron-plugin-api') + + def __call__(self): + ctxt = super(APIIdentityServiceContext, self).__call__() + if not ctxt: + return + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt['region'] = rdata.get('region') + if ctxt['region']: + return ctxt + return ctxt + diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 1c892304..233e5313 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -71,9 +71,7 @@ DVR_RESOURCE_MAP = OrderedDict([ (NEUTRON_METADATA_AGENT_CONF, { 'services': ['neutron-metadata-agent'], 'contexts': [neutron_ovs_context.DVRSharedSecretContext(), - context.NetworkServiceContext( - rel_name='neutron-network-service' - )], + neutron_ovs_context.APIIdentityServiceContext()], }), ]) TEMPLATES = 'templates/' diff --git a/metadata.yaml b/metadata.yaml index 790cda80..0e840258 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -20,8 +20,6 @@ provides: neutron-plugin: interface: neutron-plugin scope: container - neutron-network-service: - interface: quantum requires: amqp: interface: rabbitmq diff --git a/templates/juno/metadata_agent.ini b/templates/juno/metadata_agent.ini index 3062d697..c64d057c 100644 --- a/templates/juno/metadata_agent.ini +++ b/templates/juno/metadata_agent.ini @@ -4,12 +4,13 @@ ############################################################################### # Metadata service seems to cache neutron api url from keystone so trigger # restart if it changes: {{ quantum_url }} + [DEFAULT] -auth_url = {{ service_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 +auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0 auth_region = {{ region }} -admin_tenant_name = {{ service_tenant }} -admin_user = {{ service_username }} -admin_password = {{ service_password }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} root_helper = sudo neutron-rootwrap /etc/neutron/rootwrap.conf state_path = /var/lib/neutron # Gateway runs a metadata API server locally From 3161aaa4e5c3386ad71d2d2b577c877e123facec Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 1 Apr 2015 09:41:06 +0100 Subject: [PATCH 59/61] Sortout topic registration --- hooks/neutron_ovs_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index f612d1dc..54ba0501 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -72,6 +72,6 @@ def get_topics(): topics.append('q-agent-notifier-tunnel-update') topics.append('q-agent-notifier-security_group-update') topics.append('q-agent-notifier-dvr-update') - if neutron_ovs_context.neutron_api_settings()['l2_population']: + if context.NeutronAPIContext()()['l2_population']: topics.append('q-agent-notifier-l2population-update') return topics From 83d814c810a69d7a1ba8ac4f0ff3e7d4303ad7c7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2015 09:57:10 +0100 Subject: [PATCH 60/61] Fix up lint and unit tests. Unit tests needed fixing as _ensure_bridge has been removed from the OVSPluginContext in favour of doing it explicitly in configure_ovs() --- hooks/neutron_ovs_context.py | 2 +- unit_tests/test_neutron_ovs_context.py | 23 +++++++++-------------- unit_tests/test_neutron_ovs_utils.py | 6 +++--- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index be1a7839..4ca582f9 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -122,6 +122,7 @@ class DVRSharedSecretContext(OSContextGenerator): ctxt = {} return ctxt + class APIIdentityServiceContext(context.IdentityServiceContext): def __init__(self): @@ -139,4 +140,3 @@ class APIIdentityServiceContext(context.IdentityServiceContext): if ctxt['region']: return ctxt return ctxt - diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 93fab0ff..72fb5543 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -4,15 +4,11 @@ from test_utils import patch_open from mock import patch import neutron_ovs_context as context import charmhelpers + TO_PATCH = [ 'resolve_address', 'config', 'unit_get', - 'add_bridge', - 'add_bridge_port', - 'service_running', - 'service_start', - 'service_restart', 'get_host_ip', ] @@ -44,8 +40,10 @@ class OVSPluginContextTest(CharmTestCase): self.test_config.set('data-port', 'br-data:em1') config.side_effect = self.test_config.get mock_resolve_ports.side_effect = lambda ports: ports - self.assertEquals(context.DataPortContext()(), - {'br-data': 'em1'}) + self.assertEquals( + charmhelpers.contrib.openstack.context.DataPortContext()(), + {'br-data': 'em1'} + ) @patch('charmhelpers.contrib.openstack.context.config') @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') @@ -62,8 +60,10 @@ class OVSPluginContextTest(CharmTestCase): config.side_effect = self.test_config.get list_nics.return_value = machine_machs.keys() get_nic_hwaddr.side_effect = lambda nic: machine_machs[nic] - self.assertEquals(context.DataPortContext()(), - {'br-d2': 'em1'}) + self.assertEquals( + charmhelpers.contrib.openstack.context.DataPortContext()(), + {'br-d2': 'em1'} + ) @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') @@ -101,7 +101,6 @@ class OVSPluginContextTest(CharmTestCase): } _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' - self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() expect = { 'neutron_alchemy_flags': {}, @@ -125,7 +124,6 @@ class OVSPluginContextTest(CharmTestCase): 'vlan_ranges': 'physnet1:1000:2000', } self.assertEquals(expect, napi_ctxt()) - self.service_start.assertCalled() @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') @@ -168,7 +166,6 @@ class OVSPluginContextTest(CharmTestCase): } _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' - self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() expect = { 'distributed_routing': False, @@ -192,7 +189,6 @@ class OVSPluginContextTest(CharmTestCase): 'vlan_ranges': 'physnet1:1000:2000', } self.assertEquals(expect, napi_ctxt()) - self.service_start.assertCalled() class L3AgentContextTest(CharmTestCase): @@ -272,7 +268,6 @@ class DVRSharedSecretContext(CharmTestCase): _NeutronAPIContext): _NeutronAPIContext.side_effect = fake_context({'enable_dvr': True}) _shared_secret.return_value = 'secret_thing' - #_use_dvr.return_value = True self.resolve_address.return_value = '10.0.0.10' self.assertEquals(context.DVRSharedSecretContext()(), {'shared_secret': 'secret_thing', diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index abe5a3a5..dd1c5328 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -22,8 +22,8 @@ TO_PATCH = [ 'os_release', 'neutron_plugin_attribute', 'full_restart', - 'service_running', 'service_restart', + 'service_running', 'ExternalPortContext', ] @@ -111,7 +111,7 @@ class TestNeutronOVSUtils(CharmTestCase): _use_dvr.return_value = True _map = nutils.resource_map() svcs = ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent', - 'neutron-vpn-agent'] + 'neutron-l3-agent'] confs = [nutils.NEUTRON_CONF] [self.assertIn(q_conf, _map.keys()) for q_conf in confs] self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) @@ -187,7 +187,7 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'use_dvr') @patch('charmhelpers.contrib.openstack.context.config') def test_configure_ovs_ovs_ext_port(self, mock_config, _use_dvr): - _use_dvr.return_value = False + _use_dvr.return_value = True mock_config.side_effect = self.test_config.get self.config.side_effect = self.test_config.get self.test_config.set('ext-port', 'eth0') From c719eb2dd2681eb49c92d701a3059adaaeb94dff Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2015 12:28:07 +0100 Subject: [PATCH 61/61] Fix template header --- templates/juno/ml2_conf.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/juno/ml2_conf.ini b/templates/juno/ml2_conf.ini index 7cf2706a..f798463a 100644 --- a/templates/juno/ml2_conf.ini +++ b/templates/juno/ml2_conf.ini @@ -1,4 +1,4 @@ -# icehouse +# juno ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten.