From bda77fa74dc2004838f966000cdb655a6469fff0 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 14:08:36 +0100 Subject: [PATCH 1/5] Add support of os-data-network --- charm-helpers-sync.yaml | 3 +- config.yaml | 11 ++++ .../charmhelpers/contrib/hahelpers/cluster.py | 5 +- .../contrib/openstack/amulet/utils.py | 58 +++++++++++++++++++ .../charmhelpers/contrib/openstack/context.py | 24 ++++---- .../contrib/openstack/templating.py | 37 ++++++------ .../contrib/storage/linux/ceph.py | 2 +- hooks/charmhelpers/core/hookenv.py | 9 +-- hooks/charmhelpers/core/host.py | 12 ++-- hooks/charmhelpers/fetch/__init__.py | 38 +++++++----- hooks/quantum_contexts.py | 5 +- 11 files changed, 146 insertions(+), 58 deletions(-) diff --git a/charm-helpers-sync.yaml b/charm-helpers-sync.yaml index 123d4de0..884ee2ba 100644 --- a/charm-helpers-sync.yaml +++ b/charm-helpers-sync.yaml @@ -1,4 +1,4 @@ -branch: lp:charm-helpers +branch: lp:~james-page/charm-helpers/network-splits destination: hooks/charmhelpers include: - core @@ -8,3 +8,4 @@ include: - contrib.network.ovs - contrib.storage.linux - payload.execd + - contrib.network.ip diff --git a/config.yaml b/config.yaml index 591093dc..4012d97f 100644 --- a/config.yaml +++ b/config.yaml @@ -86,3 +86,14 @@ options: default: nova type: string description: Database name + # Network configuration options + # by default all access is over 'private-address' + os-data-network: + type: string + description: | + The IP address and netmask of the OpenStack Data network (e.g., + 192.168.0.0/24) + . + This network will be used for tenant network traffic in overlay + networks. + diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index bf832f7d..dd89f347 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -163,13 +163,14 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip'): +def canonical_url(configs, vip_setting='vip', address=None): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. :configs : OSTemplateRenderer: A config tempating object to inspect for a complete https context. + :vip_setting: str: Setting in charm config that specifies VIP address. ''' @@ -179,5 +180,5 @@ def canonical_url(configs, vip_setting='vip'): if is_clustered(): addr = config_get(vip_setting) else: - addr = unit_get('private-address') + addr = address or unit_get('private-address') return '%s://%s' % (scheme, addr) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 44b8b543..6515f907 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -1,4 +1,7 @@ import logging +import os +import time +import urllib import glanceclient.v1.client as glance_client import keystoneclient.v2_0 as keystone_client @@ -149,3 +152,58 @@ class OpenStackAmuletUtils(AmuletUtils): endpoint_type='publicURL') return nova_client.Client(username=user, api_key=password, project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + + if not os.path.exists(cirros_img): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, cirros_img) + f.close() + + with open(cirros_img) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + glance.images.delete(image) + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status == 'BUILD' and count < 10: + time.sleep(5) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status == 'BUILD': + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + nova.servers.delete(instance) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index e0526d89..aea11b34 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -340,10 +340,12 @@ class CephContext(OSContextGenerator): use_syslog = str(config('use-syslog')).lower() for rid in relation_ids('ceph'): for unit in related_units(rid): - mon_hosts.append(relation_get('private-address', rid=rid, - unit=unit)) auth = relation_get('auth', rid=rid, unit=unit) key = relation_get('key', rid=rid, unit=unit) + ceph_addr = \ + relation_get('ceph-public-address', rid=rid, unit=unit) or \ + relation_get('private-address', rid=rid, unit=unit) + mon_hosts.append(ceph_addr) ctxt = { 'mon_hosts': ' '.join(mon_hosts), @@ -426,12 +428,13 @@ class ApacheSSLContext(OSContextGenerator): """ Generates a context for an apache vhost configuration that configures HTTPS reverse proxying for one or many endpoints. Generated context - looks something like: - { - 'namespace': 'cinder', - 'private_address': 'iscsi.mycinderhost.com', - 'endpoints': [(8776, 8766), (8777, 8767)] - } + looks something like:: + + { + 'namespace': 'cinder', + 'private_address': 'iscsi.mycinderhost.com', + 'endpoints': [(8776, 8766), (8777, 8767)] + } The endpoints list consists of a tuples mapping external ports to internal ports. @@ -641,7 +644,7 @@ class SubordinateConfigContext(OSContextGenerator): The subordinate interface allows subordinates to export their configuration requirements to the principle for multiple config files and multiple serivces. Ie, a subordinate that has interfaces - to both glance and nova may export to following yaml blob as json: + to both glance and nova may export to following yaml blob as json:: glance: /etc/glance/glance-api.conf: @@ -660,7 +663,8 @@ class SubordinateConfigContext(OSContextGenerator): It is then up to the principle charms to subscribe this context to the service+config file it is interestd in. Configuration data will - be available in the template context, in glance's case, as: + be available in the template context, in glance's case, as:: + ctxt = { ... other context ... 'subordinate_config': { diff --git a/hooks/charmhelpers/contrib/openstack/templating.py b/hooks/charmhelpers/contrib/openstack/templating.py index 4595778c..f5442712 100644 --- a/hooks/charmhelpers/contrib/openstack/templating.py +++ b/hooks/charmhelpers/contrib/openstack/templating.py @@ -30,17 +30,17 @@ def get_loader(templates_dir, os_release): loading dir. A charm may also ship a templates dir with this module - and it will be appended to the bottom of the search list, eg: - hooks/charmhelpers/contrib/openstack/templates. + and it will be appended to the bottom of the search list, eg:: - :param templates_dir: str: Base template directory containing release - sub-directories. - :param os_release : str: OpenStack release codename to construct template - loader. + hooks/charmhelpers/contrib/openstack/templates - :returns : jinja2.ChoiceLoader constructed with a list of - jinja2.FilesystemLoaders, ordered in descending - order by OpenStack release. + :param templates_dir (str): Base template directory containing release + sub-directories. + :param os_release (str): OpenStack release codename to construct template + loader. + :returns: jinja2.ChoiceLoader constructed with a list of + jinja2.FilesystemLoaders, ordered in descending + order by OpenStack release. """ tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) for rel in OPENSTACK_CODENAMES.itervalues()] @@ -111,7 +111,8 @@ class OSConfigRenderer(object): and ease the burden of managing config templates across multiple OpenStack releases. - Basic usage: + Basic usage:: + # import some common context generates from charmhelpers from charmhelpers.contrib.openstack import context @@ -131,21 +132,19 @@ class OSConfigRenderer(object): # write out all registered configs configs.write_all() - Details: + **OpenStack Releases and template loading** - OpenStack Releases and template loading - --------------------------------------- When the object is instantiated, it is associated with a specific OS release. This dictates how the template loader will be constructed. The constructed loader attempts to load the template from several places in the following order: - - from the most recent OS release-specific template dir (if one exists) - - the base templates_dir - - a template directory shipped in the charm with this helper file. + - from the most recent OS release-specific template dir (if one exists) + - the base templates_dir + - a template directory shipped in the charm with this helper file. + For the example above, '/tmp/templates' contains the following structure:: - For the example above, '/tmp/templates' contains the following structure: /tmp/templates/nova.conf /tmp/templates/api-paste.ini /tmp/templates/grizzly/api-paste.ini @@ -169,8 +168,8 @@ class OSConfigRenderer(object): $CHARM/hooks/charmhelpers/contrib/openstack/templates. This allows us to ship common templates (haproxy, apache) with the helpers. - Context generators - --------------------------------------- + **Context generators** + Context generators are used to generate template contexts during hook execution. Doing so may require inspecting service relations, charm config, etc. When registered, a config file is associated with a list diff --git a/hooks/charmhelpers/contrib/storage/linux/ceph.py b/hooks/charmhelpers/contrib/storage/linux/ceph.py index 12417410..768438a4 100644 --- a/hooks/charmhelpers/contrib/storage/linux/ceph.py +++ b/hooks/charmhelpers/contrib/storage/linux/ceph.py @@ -303,7 +303,7 @@ def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, blk_device, fstype, system_services=[]): """ NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. + the same rbd_img otherwise data loss will occur. Ensures given pool and RBD image exists, is mapped to a block device, and the device is formatted and mounted at the given mount_point. diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index c2e66f66..c9530433 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -25,7 +25,7 @@ cache = {} def cached(func): """Cache return values for multiple executions of func + args - For example: + For example:: @cached def unit_get(attribute): @@ -445,18 +445,19 @@ class UnregisteredHookError(Exception): class Hooks(object): """A convenient handler for hook functions. - Example: + Example:: + hooks = Hooks() # register a hook, taking its name from the function name @hooks.hook() def install(): - ... + pass # your code here # register a hook, providing a custom hook name @hooks.hook("config-changed") def config_changed(): - ... + pass # your code here if __name__ == "__main__": # execute a hook based on the name the program is called by diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 59f8facc..8b617a42 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -211,13 +211,13 @@ def file_hash(path): def restart_on_change(restart_map, stopstart=False): """Restart services based on configuration files changing - This function is used a decorator, for example + This function is used a decorator, for example:: @restart_on_change({ '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] }) def ceph_client_changed(): - ... + pass # your code here In this example, the cinder-api and cinder-volume services would be restarted if /etc/ceph/ceph.conf is changed by the @@ -313,9 +313,11 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): '''Compare supplied revno with the revno of the installed package - 1 => Installed revno is greater than supplied arg - 0 => Installed revno is the same as supplied arg - -1 => Installed revno is less than supplied arg + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + ''' import apt_pkg if not pkgcache: diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index b5cb48ef..5be512ce 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -235,31 +235,39 @@ def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): """ - Configure multiple sources from charm configuration + Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The frament needs to be included as a string. Example config: - install_sources: + install_sources: | - "ppa:foo" - "http://example.com/repo precise main" - install_keys: + install_keys: | - null - "a1b2c3d4" Note that 'null' (a.k.a. None) should not be quoted. """ - sources = safe_load(config(sources_var)) - keys = config(keys_var) - if keys is not None: - keys = safe_load(keys) - if isinstance(sources, basestring) and ( - keys is None or isinstance(keys, basestring)): - add_source(sources, keys) + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, basestring): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) else: - if not len(sources) == len(keys): - msg = 'Install sources and keys lists are different lengths' - raise SourceConfigError(msg) - for src_num in range(len(sources)): - add_source(sources[src_num], keys[src_num]) + if isinstance(keys, basestring): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) if update: apt_update(fatal=True) diff --git a/hooks/quantum_contexts.py b/hooks/quantum_contexts.py index c8b635aa..c13a8c2c 100644 --- a/hooks/quantum_contexts.py +++ b/hooks/quantum_contexts.py @@ -28,6 +28,7 @@ from charmhelpers.contrib.hahelpers.cluster import( eligible_leader ) import re +from charmhelpers.contrib.network.ip import get_address_in_network DB_USER = "quantum" QUANTUM_DB = "quantum" @@ -164,7 +165,9 @@ class QuantumGatewayContext(OSContextGenerator): def __call__(self): ctxt = { 'shared_secret': get_shared_secret(), - 'local_ip': get_host_ip(), # XXX: data network impact + 'local_ip': + get_address_in_network(config('os-data-network'), + get_host_ip(unit_get('private-address'))), 'core_plugin': core_plugin(), 'plugin': config('plugin'), 'debug': config('debug'), From 8fecafa335b84179ef44cbd8b5d6e1afa6a38f84 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 27 Jun 2014 14:13:23 +0100 Subject: [PATCH 2/5] Add missing charmhelper --- hooks/charmhelpers/contrib/network/ip.py | 69 ++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 hooks/charmhelpers/contrib/network/ip.py diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 00000000..15a6731c --- /dev/null +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,69 @@ +import sys + +from charmhelpers.fetch import apt_install +from charmhelpers.core.hookenv import ( + ERROR, log, +) + +try: + import netifaces +except ImportError: + apt_install('python-netifaces') + import netifaces + +try: + import netaddr +except ImportError: + apt_install('python-netaddr') + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def get_address_in_network(network, fallback=None, fatal=False): + """ + Get an IPv4 address within the network from the host. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + + """ + + def not_found_error_out(): + log("No IP address found in network: %s" % network, + level=ERROR) + sys.exit(1) + + if network is None: + if fallback is not None: + return fallback + else: + if fatal: + not_found_error_out() + + _validate_cidr(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if cidr in netaddr.IPNetwork(network): + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + not_found_error_out() + + return None From 8132d4a8a5fef7c74632a0b84743d32c2559e10c Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 09:25:12 +0100 Subject: [PATCH 3/5] Resync helpers --- .bzrignore | 1 + Makefile | 15 ++++++---- charm-helpers-sync.yaml => charm-helpers.yaml | 0 hooks/charmhelpers/contrib/network/ip.py | 26 +++++++++++++++++ .../contrib/openstack/amulet/deployment.py | 29 +++++++++++++++---- .../contrib/openstack/amulet/utils.py | 10 +++---- .../charmhelpers/contrib/openstack/context.py | 17 +++++++++++ 7 files changed, 81 insertions(+), 17 deletions(-) rename charm-helpers-sync.yaml => charm-helpers.yaml (100%) diff --git a/.bzrignore b/.bzrignore index 6350e986..a2c7a097 100644 --- a/.bzrignore +++ b/.bzrignore @@ -1 +1,2 @@ +bin .coverage diff --git a/Makefile b/Makefile index 0e79dbb5..8dc88152 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,19 @@ PYTHON := /usr/bin/env python lint: - @flake8 --exclude hooks/charmhelpers hooks - @flake8 --exclude hooks/charmhelpers unit_tests + @flake8 --exclude hooks/charmhelpers hooks unit_tests @charm proof test: - @echo Starting tests... - @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests + @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests -sync: - @charm-helper-sync -c charm-helpers-sync.yaml +bin/charm_helpers_sync.py: + @mkdir -p bin + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ + > bin/charm_helpers_sync.py + +sync: bin/charm_helpers_sync.py + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml publish: lint test bzr push lp:charms/quantum-gateway diff --git a/charm-helpers-sync.yaml b/charm-helpers.yaml similarity index 100% rename from charm-helpers-sync.yaml rename to charm-helpers.yaml diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index 15a6731c..f2fa263f 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -67,3 +67,29 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() return None + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + if address in network: + return True + else: + return False diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 9e164821..e476b6f2 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -7,19 +7,36 @@ class OpenStackAmuletDeployment(AmuletDeployment): """This class inherits from AmuletDeployment and has additional support that is specifically for use by OpenStack charms.""" - def __init__(self, series=None, openstack=None): + def __init__(self, series=None, openstack=None, source=None): """Initialize the deployment environment.""" - self.openstack = None super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source - if openstack: - self.openstack = openstack + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin.""" + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + name = 0 + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] + + if self.openstack: + for svc in services: + if svc[name] not in use_source: + config = {'openstack-origin': self.openstack} + self.d.configure(svc[name], config) + + if self.source: + for svc in services: + if svc[name] in use_source: + config = {'source': self.source} + self.d.configure(svc[name], config) def _configure_services(self, configs): """Configure all of the services.""" for service, config in configs.iteritems(): - if service == self.this_service: - config['openstack-origin'] = self.openstack self.d.configure(service, config) def _get_openstack_release(self): diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 6515f907..222281e3 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -74,7 +74,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected tenant data - {}".format(ret) if not found: - return "tenant {} does not exist".format(e.name) + return "tenant {} does not exist".format(e['name']) return ret def validate_role_data(self, expected, actual): @@ -91,7 +91,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected role data - {}".format(ret) if not found: - return "role {} does not exist".format(e.name) + return "role {} does not exist".format(e['name']) return ret def validate_user_data(self, expected, actual): @@ -110,7 +110,7 @@ class OpenStackAmuletUtils(AmuletUtils): if ret: return "unexpected user data - {}".format(ret) if not found: - return "user {} does not exist".format(e.name) + return "user {} does not exist".format(e['name']) return ret def validate_flavor_data(self, expected, actual): @@ -192,8 +192,8 @@ class OpenStackAmuletUtils(AmuletUtils): count = 1 status = instance.status - while status == 'BUILD' and count < 10: - time.sleep(5) + while status != 'ACTIVE' and count < 60: + time.sleep(3) instance = nova.servers.get(instance.id) status = instance.status self.log.debug('instance status: {}'.format(status)) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index aea11b34..b21fca60 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -21,6 +21,7 @@ from charmhelpers.core.hookenv import ( relation_get, relation_ids, related_units, + relation_set, unit_get, unit_private_ip, ERROR, @@ -42,6 +43,8 @@ from charmhelpers.contrib.openstack.neutron import ( neutron_plugin_attribute, ) +from charmhelpers.contrib.network.ip import get_address_in_network + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' @@ -134,8 +137,22 @@ class SharedDBContext(OSContextGenerator): 'Missing required charm config options. ' '(database name and user)') raise OSContextError + ctxt = {} + # NOTE(jamespage) if mysql charm provides a network upon which + # access to the database should be made, reconfigure relation + # with the service units local address and defer execution + access_network = relation_get('access-network') + if access_network is not None: + access_hostname = get_address_in_network(access_network, + unit_get('private-address')) + set_hostname = relation_get(attribute='hostname', + unit=local_unit()) + if set_hostname != access_hostname: + relation_set(hostname=access_hostname) + return ctxt # Defer any further hook execution for now.... + password_setting = 'password' if self.relation_prefix: password_setting = self.relation_prefix + '_password' From 62dd6aa7dd96c249e95d95114bbda53edde0e961 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 2 Jul 2014 10:40:59 +0100 Subject: [PATCH 4/5] Resync helpers --- .../charmhelpers/contrib/openstack/context.py | 8 ++++-- hooks/charmhelpers/contrib/openstack/vip.py | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index b21fca60..9da9c1ff 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -145,12 +145,16 @@ class SharedDBContext(OSContextGenerator): # with the service units local address and defer execution access_network = relation_get('access-network') if access_network is not None: + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) - set_hostname = relation_get(attribute='hostname', + set_hostname = relation_get(attribute=hostname_key, unit=local_unit()) if set_hostname != access_hostname: - relation_set(hostname=access_hostname) + relation_set(relation_settings={hostname_key: access_hostname}) return ctxt # Defer any further hook execution for now.... password_setting = 'password' diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py new file mode 100644 index 00000000..d8c42f90 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/vip.py @@ -0,0 +1,25 @@ + +from netaddr import IPAddress, IPNetwork + +class VIPConfiguration(): + + def __init__(self, configuration): + self.vip = [] + for vip in configuration.split(): + self.vips.append(IPAddress(vip)) + + def getVIP(self, network): + ''' Determine the VIP for the provided network + :network str: CIDR presented network, e.g. 192.168.1.1/24 + :returns str: IP address of VIP in provided network or None + ''' + network = IPNetwork(network) + for vip in self.vips: + if vip in network: + return str(vip) + return None + + def getNIC(self, network): + ''' Determine the physical network interface in use + for the specified network''' + From 32e85a148fb3f49d2b3b9d9f8a35425f32882b0f Mon Sep 17 00:00:00 2001 From: James Page Date: Thu, 24 Jul 2014 11:33:09 +0100 Subject: [PATCH 5/5] Rebase on trunk helpers --- charm-helpers.yaml | 2 +- .../charmhelpers/contrib/hahelpers/cluster.py | 8 +-- hooks/charmhelpers/contrib/network/ip.py | 67 ++++++++++++++++++- .../contrib/network/ovs/__init__.py | 7 +- .../charmhelpers/contrib/openstack/context.py | 19 ++++-- hooks/charmhelpers/contrib/openstack/vip.py | 25 ------- hooks/charmhelpers/core/host.py | 4 ++ 7 files changed, 92 insertions(+), 40 deletions(-) delete mode 100644 hooks/charmhelpers/contrib/openstack/vip.py diff --git a/charm-helpers.yaml b/charm-helpers.yaml index 884ee2ba..81810093 100644 --- a/charm-helpers.yaml +++ b/charm-helpers.yaml @@ -1,4 +1,4 @@ -branch: lp:~james-page/charm-helpers/network-splits +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py index dd89f347..505de6b2 100644 --- a/hooks/charmhelpers/contrib/hahelpers/cluster.py +++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py @@ -146,12 +146,12 @@ def get_hacluster_config(): Obtains all relevant configuration from charm configuration required for initiating a relation to hacluster: - ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr + ha-bindiface, ha-mcastport, vip returns: dict: A dict containing settings keyed by setting name. raises: HAIncompleteConfig if settings are missing. ''' - settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr'] + settings = ['ha-bindiface', 'ha-mcastport', 'vip'] conf = {} for setting in settings: conf[setting] = config_get(setting) @@ -163,7 +163,7 @@ def get_hacluster_config(): return conf -def canonical_url(configs, vip_setting='vip', address=None): +def canonical_url(configs, vip_setting='vip'): ''' Returns the correct HTTP URL to this host given the state of HTTPS configuration and hacluster. @@ -180,5 +180,5 @@ def canonical_url(configs, vip_setting='vip', address=None): if is_clustered(): addr = config_get(vip_setting) else: - addr = address or unit_get('private-address') + addr = unit_get('private-address') return '%s://%s' % (scheme, addr) diff --git a/hooks/charmhelpers/contrib/network/ip.py b/hooks/charmhelpers/contrib/network/ip.py index f2fa263f..0972e91a 100644 --- a/hooks/charmhelpers/contrib/network/ip.py +++ b/hooks/charmhelpers/contrib/network/ip.py @@ -1,5 +1,7 @@ import sys +from functools import partial + from charmhelpers.fetch import apt_install from charmhelpers.core.hookenv import ( ERROR, log, @@ -28,7 +30,7 @@ def _validate_cidr(network): def get_address_in_network(network, fallback=None, fatal=False): """ - Get an IPv4 address within the network from the host. + Get an IPv4 or IPv6 address within the network from the host. :param network (str): CIDR presentation format. For example, '192.168.1.0/24'. @@ -51,14 +53,22 @@ def get_address_in_network(network, fallback=None, fatal=False): not_found_error_out() _validate_cidr(network) + network = netaddr.IPNetwork(network) for iface in netifaces.interfaces(): addresses = netifaces.ifaddresses(iface) - if netifaces.AF_INET in addresses: + if network.version == 4 and netifaces.AF_INET in addresses: addr = addresses[netifaces.AF_INET][0]['addr'] netmask = addresses[netifaces.AF_INET][0]['netmask'] cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) - if cidr in netaddr.IPNetwork(network): + if cidr in network: return str(cidr.ip) + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) if fallback is not None: return fallback @@ -69,6 +79,17 @@ def get_address_in_network(network, fallback=None, fatal=False): return None +def is_ipv6(address): + '''Determine whether provided address is IPv6 or not''' + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + else: + return address.version == 6 + + def is_address_in_network(network, address): """ Determine whether the provided address is within a network range. @@ -93,3 +114,43 @@ def is_address_in_network(network, address): return True else: return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + if not addr['addr'].startswith('fe80'): + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if address in cidr: + if key == 'iface': + return iface + else: + return addr[key] + return None + + +get_iface_for_address = partial(_get_for_address, key='iface') + +get_netmask_for_address = partial(_get_for_address, key='netmask') diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index 5eba8376..8f8a5230 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -21,12 +21,16 @@ def del_bridge(name): subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name]) -def add_bridge_port(name, port): +def add_bridge_port(name, port, promisc=False): ''' Add a port to the named openvswitch bridge ''' log('Adding port {} to bridge {}'.format(port, name)) subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-port", name, port]) subprocess.check_call(["ip", "link", "set", port, "up"]) + if promisc: + subprocess.check_call(["ip", "link", "set", port, "promisc", "on"]) + else: + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) def del_bridge_port(name, port): @@ -35,6 +39,7 @@ def del_bridge_port(name, port): subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port", name, port]) subprocess.check_call(["ip", "link", "set", port, "down"]) + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) def set_manager(manager): diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 9da9c1ff..92c41b23 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -25,6 +25,7 @@ from charmhelpers.core.hookenv import ( unit_get, unit_private_ip, ERROR, + INFO ) from charmhelpers.contrib.hahelpers.cluster import ( @@ -148,7 +149,7 @@ class SharedDBContext(OSContextGenerator): if self.relation_prefix is not None: hostname_key = "{}_hostname".format(self.relation_prefix) else: - hostname_key = "hostname" + hostname_key = "hostname" access_hostname = get_address_in_network(access_network, unit_get('private-address')) set_hostname = relation_get(attribute=hostname_key, @@ -400,7 +401,9 @@ class HAProxyContext(OSContextGenerator): cluster_hosts = {} l_unit = local_unit().replace('/', '-') - cluster_hosts[l_unit] = unit_get('private-address') + cluster_hosts[l_unit] = \ + get_address_in_network(config('os-internal-network'), + unit_get('private-address')) for rid in relation_ids('cluster'): for unit in related_units(rid): @@ -712,7 +715,7 @@ class SubordinateConfigContext(OSContextGenerator): self.interface = interface def __call__(self): - ctxt = {} + ctxt = {'sections': {}} for rid in relation_ids(self.interface): for unit in related_units(rid): sub_config = relation_get('subordinate_configuration', @@ -738,10 +741,14 @@ class SubordinateConfigContext(OSContextGenerator): sub_config = sub_config[self.config_file] for k, v in sub_config.iteritems(): - ctxt[k] = v + if k == 'sections': + for section, config_dict in v.iteritems(): + log("adding section '%s'" % (section)) + ctxt[k][section] = config_dict + else: + ctxt[k] = v - if not ctxt: - ctxt['sections'] = {} + log("%d section(s) found" % (len(ctxt['sections'])), level=INFO) return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/vip.py b/hooks/charmhelpers/contrib/openstack/vip.py deleted file mode 100644 index d8c42f90..00000000 --- a/hooks/charmhelpers/contrib/openstack/vip.py +++ /dev/null @@ -1,25 +0,0 @@ - -from netaddr import IPAddress, IPNetwork - -class VIPConfiguration(): - - def __init__(self, configuration): - self.vip = [] - for vip in configuration.split(): - self.vips.append(IPAddress(vip)) - - def getVIP(self, network): - ''' Determine the VIP for the provided network - :network str: CIDR presented network, e.g. 192.168.1.1/24 - :returns str: IP address of VIP in provided network or None - ''' - network = IPNetwork(network) - for vip in self.vips: - if vip in network: - return str(vip) - return None - - def getNIC(self, network): - ''' Determine the physical network interface in use - for the specified network''' - diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 8b617a42..d934f940 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -322,6 +322,10 @@ def cmp_pkgrevno(package, revno, pkgcache=None): import apt_pkg if not pkgcache: apt_pkg.init() + # Force Apt to build its cache in memory. That way we avoid race + # conditions with other applications building the cache in the same + # place. + apt_pkg.config.set("Dir::Cache::pkgcache", "") pkgcache = apt_pkg.Cache() pkg = pkgcache[package] return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)