diff --git a/hooks/charmhelpers/contrib/hahelpers/cluster.py b/hooks/charmhelpers/contrib/hahelpers/cluster.py
index c555d7aa..aa0b515d 100644
--- a/hooks/charmhelpers/contrib/hahelpers/cluster.py
+++ b/hooks/charmhelpers/contrib/hahelpers/cluster.py
@@ -44,6 +44,7 @@ from charmhelpers.core.hookenv import (
ERROR,
WARNING,
unit_get,
+ is_leader as juju_is_leader
)
from charmhelpers.core.decorators import (
retry_on_exception,
@@ -63,17 +64,30 @@ class CRMResourceNotFound(Exception):
pass
+class CRMDCNotFound(Exception):
+ pass
+
+
def is_elected_leader(resource):
"""
Returns True if the charm executing this is the elected cluster leader.
It relies on two mechanisms to determine leadership:
- 1. If the charm is part of a corosync cluster, call corosync to
+ 1. If juju is sufficiently new and leadership election is supported,
+ the is_leader command will be used.
+ 2. If the charm is part of a corosync cluster, call corosync to
determine leadership.
- 2. If the charm is not part of a corosync cluster, the leader is
+ 3. If the charm is not part of a corosync cluster, the leader is
determined as being "the alive unit with the lowest unit numer". In
other words, the oldest surviving unit.
"""
+ try:
+ return juju_is_leader()
+ except NotImplementedError:
+ log('Juju leadership election feature not enabled'
+ ', using fallback support',
+ level=WARNING)
+
if is_clustered():
if not is_crm_leader(resource):
log('Deferring action to CRM leader.', level=INFO)
@@ -106,8 +120,9 @@ def is_crm_dc():
status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if not isinstance(status, six.text_type):
status = six.text_type(status, "utf-8")
- except subprocess.CalledProcessError:
- return False
+ except subprocess.CalledProcessError as ex:
+ raise CRMDCNotFound(str(ex))
+
current_dc = ''
for line in status.split('\n'):
if line.startswith('Current DC'):
@@ -115,10 +130,14 @@ def is_crm_dc():
current_dc = line.split(':')[1].split()[0]
if current_dc == get_unit_hostname():
return True
+ elif current_dc == 'NONE':
+ raise CRMDCNotFound('Current DC: NONE')
+
return False
-@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)
+@retry_on_exception(5, base_delay=2,
+ exc_type=(CRMResourceNotFound, CRMDCNotFound))
def is_crm_leader(resource, retry=False):
"""
Returns True if the charm calling this is the elected corosync leader,
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
index 461a702f..c664c9d0 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
- self.trusty_kilo, self.vivid_kilo) = range(10)
+ self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+ self.wily_liberty) = range(12)
releases = {
('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+ ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('utopic', None): self.utopic_juno,
- ('vivid', None): self.vivid_kilo}
+ ('vivid', None): self.vivid_kilo,
+ ('wily', None): self.wily_liberty}
return releases[(self.series, self.openstack)]
def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
+ ('wily', 'liberty'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index 9c3d918a..576bf0b5 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -16,15 +16,15 @@
import logging
import os
+import six
import time
import urllib
import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
-import six
-
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
This class inherits from AmuletUtils and has additional support
- that is specifically for use by OpenStack charms.
+ that is specifically for use by OpenStack charm tests.
"""
def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate actual endpoint data vs expected endpoint data. The ports
are used to find the matching endpoint.
"""
+ self.log.debug('Validating endpoint data...')
+ self.log.debug('actual: {}'.format(repr(endpoints)))
found = False
for ep in endpoints:
self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual service catalog endpoints vs a list of
expected service catalog endpoints.
"""
+ self.log.debug('Validating service catalog endpoint data...')
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in six.iteritems(expected):
if k in actual:
@@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual tenant data vs list of expected tenant
data.
"""
+ self.log.debug('Validating tenant data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual role data vs a list of expected role
data.
"""
+ self.log.debug('Validating role data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual user data vs a list of expected user
data.
"""
+ self.log.debug('Validating user data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual flavors vs a list of expected flavors.
"""
+ self.log.debug('Validating flavor data...')
self.log.debug('actual: {}'.format(repr(actual)))
act = [a.name for a in actual]
return self._validate_list_data(expected, act)
def tenant_exists(self, keystone, tenant):
"""Return True if tenant exists."""
+ self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant):
"""Authenticates admin user with the keystone admin endpoint."""
+ self.log.debug('Authenticating keystone admin...')
unit = keystone_sentry
service_ip = unit.relation('shared-db',
'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
+ self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_glance_admin(self, keystone):
"""Authenticates admin user with glance."""
+ self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
+ def authenticate_heat_admin(self, keystone):
+ """Authenticates the admin user with heat."""
+ self.log.debug('Authenticating heat admin...')
+ ep = keystone.service_catalog.url_for(service_type='orchestration',
+ endpoint_type='publicURL')
+ return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
+ self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance."""
+ self.log.debug('Creating glance image ({})...'.format(image_name))
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_image(self, glance, image):
"""Delete the specified image."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_image.')
+ self.log.debug('Deleting glance image ({})...'.format(image))
num_before = len(list(glance.images.list()))
glance.images.delete(image)
@@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_instance(self, nova, image_name, instance_name, flavor):
"""Create the specified instance."""
+ self.log.debug('Creating instance '
+ '({}|{}|{})'.format(instance_name, image_name, flavor))
image = nova.images.find(name=image_name)
flavor = nova.flavors.find(name=flavor)
instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_instance(self, nova, instance):
"""Delete the specified instance."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_instance.')
+ self.log.debug('Deleting instance ({})...'.format(instance))
num_before = len(list(nova.servers.list()))
nova.servers.delete(instance)
@@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
return False
return True
+
+ def create_or_get_keypair(self, nova, keypair_name="testkey"):
+ """Create a new keypair, or return pointer if it already exists."""
+ try:
+ _keypair = nova.keypairs.get(keypair_name)
+ self.log.debug('Keypair ({}) already exists, '
+ 'using it.'.format(keypair_name))
+ return _keypair
+ except:
+ self.log.debug('Keypair ({}) does not exist, '
+ 'creating it.'.format(keypair_name))
+
+ _keypair = nova.keypairs.create(name=keypair_name)
+ return _keypair
+
+ def delete_resource(self, resource, resource_id,
+ msg="resource", max_wait=120):
+ """Delete one openstack resource, such as one instance, keypair,
+ image, volume, stack, etc., and confirm deletion within max wait time.
+
+ :param resource: pointer to os resource type, ex:glance_client.images
+ :param resource_id: unique name or id for the openstack resource
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, otherwise False
+ """
+ num_before = len(list(resource.list()))
+ resource.delete(resource_id)
+
+ tries = 0
+ num_after = len(list(resource.list()))
+ while num_after != (num_before - 1) and tries < (max_wait / 4):
+ self.log.debug('{} delete check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ num_before,
+ num_after,
+ resource_id))
+ time.sleep(4)
+ num_after = len(list(resource.list()))
+ tries += 1
+
+ self.log.debug('{}: expected, actual count = {}, '
+ '{}'.format(msg, num_before - 1, num_after))
+
+ if num_after == (num_before - 1):
+ return True
+ else:
+ self.log.error('{} delete timed out'.format(msg))
+ return False
+
+ def resource_reaches_status(self, resource, resource_id,
+ expected_stat='available',
+ msg='resource', max_wait=120):
+ """Wait for an openstack resources status to reach an
+ expected status within a specified time. Useful to confirm that
+ nova instances, cinder vols, snapshots, glance images, heat stacks
+ and other resources eventually reach the expected status.
+
+ :param resource: pointer to os resource type, ex: heat_client.stacks
+ :param resource_id: unique id for the openstack resource
+ :param expected_stat: status to expect resource to reach
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, False if status is not reached
+ """
+
+ tries = 0
+ resource_stat = resource.get(resource_id).status
+ while resource_stat != expected_stat and tries < (max_wait / 4):
+ self.log.debug('{} status check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ resource_stat,
+ expected_stat,
+ resource_id))
+ time.sleep(4)
+ resource_stat = resource.get(resource_id).status
+ tries += 1
+
+ self.log.debug('{}: expected, actual status = {}, '
+ '{}'.format(msg, resource_stat, expected_stat))
+
+ if resource_stat == expected_stat:
+ return True
+ else:
+ self.log.debug('{} never reached expected status: '
+ '{}'.format(resource_id, expected_stat))
+ return False
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index 400eaf8e..ab400060 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -240,7 +240,7 @@ class SharedDBContext(OSContextGenerator):
if self.relation_prefix:
password_setting = self.relation_prefix + '_password'
- for rid in relation_ids('shared-db'):
+ for rid in relation_ids(self.interfaces[0]):
for unit in related_units(rid):
rdata = relation_get(rid=rid, unit=unit)
host = rdata.get('db_host')
diff --git a/hooks/charmhelpers/contrib/openstack/ip.py b/hooks/charmhelpers/contrib/openstack/ip.py
index 29bbddcb..3dca6dc1 100644
--- a/hooks/charmhelpers/contrib/openstack/ip.py
+++ b/hooks/charmhelpers/contrib/openstack/ip.py
@@ -17,6 +17,7 @@
from charmhelpers.core.hookenv import (
config,
unit_get,
+ service_name,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
@@ -26,8 +27,6 @@ from charmhelpers.contrib.network.ip import (
)
from charmhelpers.contrib.hahelpers.cluster import is_clustered
-from functools import partial
-
PUBLIC = 'public'
INTERNAL = 'int'
ADMIN = 'admin'
@@ -35,15 +34,18 @@ ADMIN = 'admin'
ADDRESS_MAP = {
PUBLIC: {
'config': 'os-public-network',
- 'fallback': 'public-address'
+ 'fallback': 'public-address',
+ 'override': 'os-public-hostname',
},
INTERNAL: {
'config': 'os-internal-network',
- 'fallback': 'private-address'
+ 'fallback': 'private-address',
+ 'override': 'os-internal-hostname',
},
ADMIN: {
'config': 'os-admin-network',
- 'fallback': 'private-address'
+ 'fallback': 'private-address',
+ 'override': 'os-admin-hostname',
}
}
@@ -57,15 +59,50 @@ def canonical_url(configs, endpoint_type=PUBLIC):
:param endpoint_type: str endpoint type to resolve.
:param returns: str base URL for services on the current service unit.
"""
- scheme = 'http'
- if 'https' in configs.complete_contexts():
- scheme = 'https'
+ scheme = _get_scheme(configs)
+
address = resolve_address(endpoint_type)
if is_ipv6(address):
address = "[{}]".format(address)
+
return '%s://%s' % (scheme, address)
+def _get_scheme(configs):
+ """Returns the scheme to use for the url (either http or https)
+ depending upon whether https is in the configs value.
+
+ :param configs: OSTemplateRenderer config templating object to inspect
+ for a complete https context.
+ :returns: either 'http' or 'https' depending on whether https is
+ configured within the configs context.
+ """
+ scheme = 'http'
+ if configs and 'https' in configs.complete_contexts():
+ scheme = 'https'
+ return scheme
+
+
+def _get_address_override(endpoint_type=PUBLIC):
+ """Returns any address overrides that the user has defined based on the
+ endpoint type.
+
+ Note: this function allows for the service name to be inserted into the
+ address if the user specifies {service_name}.somehost.org.
+
+ :param endpoint_type: the type of endpoint to retrieve the override
+ value for.
+ :returns: any endpoint address or hostname that the user has overridden
+ or None if an override is not present.
+ """
+ override_key = ADDRESS_MAP[endpoint_type]['override']
+ addr_override = config(override_key)
+ if not addr_override:
+ return None
+ else:
+ return addr_override.format(service_name=service_name())
+
+
def resolve_address(endpoint_type=PUBLIC):
"""Return unit address depending on net config.
@@ -77,7 +114,10 @@ def resolve_address(endpoint_type=PUBLIC):
:param endpoint_type: Network endpoing type
"""
- resolved_address = None
+ resolved_address = _get_address_override(endpoint_type)
+ if resolved_address:
+ return resolved_address
+
vips = config('vip')
if vips:
vips = vips.split()
@@ -109,38 +149,3 @@ def resolve_address(endpoint_type=PUBLIC):
"clustered=%s)" % (net_type, clustered))
return resolved_address
-
-
-def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC,
- override=None):
- """Returns the correct endpoint URL to advertise to Keystone.
-
- This method provides the correct endpoint URL which should be advertised to
- the keystone charm for endpoint creation. This method allows for the url to
- be overridden to force a keystone endpoint to have specific URL for any of
- the defined scopes (admin, internal, public).
-
- :param configs: OSTemplateRenderer config templating object to inspect
- for a complete https context.
- :param url_template: str format string for creating the url template. Only
- two values will be passed - the scheme+hostname
- returned by the canonical_url and the port.
- :param endpoint_type: str endpoint type to resolve.
- :param override: str the name of the config option which overrides the
- endpoint URL defined by the charm itself. None will
- disable any overrides (default).
- """
- if override:
- # Return any user-defined overrides for the keystone endpoint URL.
- user_value = config(override)
- if user_value:
- return user_value.strip()
-
- return url_template % (canonical_url(configs, endpoint_type), port)
-
-
-public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC)
-
-internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL)
-
-admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN)
diff --git a/hooks/charmhelpers/contrib/openstack/neutron.py b/hooks/charmhelpers/contrib/openstack/neutron.py
index b3aa3d4c..f7b72352 100644
--- a/hooks/charmhelpers/contrib/openstack/neutron.py
+++ b/hooks/charmhelpers/contrib/openstack/neutron.py
@@ -172,14 +172,16 @@ def neutron_plugins():
'services': ['calico-felix',
'bird',
'neutron-dhcp-agent',
- 'nova-api-metadata'],
+ 'nova-api-metadata',
+ 'etcd'],
'packages': [[headers_package()] + determine_dkms_package(),
['calico-compute',
'bird',
'neutron-dhcp-agent',
- 'nova-api-metadata']],
- 'server_packages': ['neutron-server', 'calico-control'],
- 'server_services': ['neutron-server']
+ 'nova-api-metadata',
+ 'etcd']],
+ 'server_packages': ['neutron-server', 'calico-control', 'etcd'],
+ 'server_services': ['neutron-server', 'etcd']
},
'vsp': {
'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index d795a358..28532c98 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -79,6 +79,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
+ ('wily', 'liberty'),
])
@@ -91,6 +92,7 @@ OPENSTACK_CODENAMES = OrderedDict([
('2014.1', 'icehouse'),
('2014.2', 'juno'),
('2015.1', 'kilo'),
+ ('2015.2', 'liberty'),
])
# The ugly duckling
@@ -113,6 +115,7 @@ SWIFT_CODENAMES = OrderedDict([
('2.2.0', 'juno'),
('2.2.1', 'kilo'),
('2.2.2', 'kilo'),
+ ('2.3.0', 'liberty'),
])
DEFAULT_LOOPBACK_SIZE = '5G'
@@ -321,6 +324,9 @@ def configure_installation_source(rel):
'kilo': 'trusty-updates/kilo',
'kilo/updates': 'trusty-updates/kilo',
'kilo/proposed': 'trusty-proposed/kilo',
+ 'liberty': 'trusty-updates/liberty',
+ 'liberty/updates': 'trusty-updates/liberty',
+ 'liberty/proposed': 'trusty-proposed/liberty',
}
try:
@@ -549,6 +555,11 @@ def git_clone_and_install(projects_yaml, core_project, depth=1):
pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
+ # Upgrade setuptools from default virtualenv version. The default version
+ # in trusty breaks update.py in global requirements master branch.
+ pip_install('setuptools', upgrade=True, proxy=http_proxy,
+ venv=os.path.join(parent_dir, 'venv'))
+
for p in projects['repositories']:
repo = p['repository']
branch = p['branch']
@@ -610,24 +621,24 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
else:
repo_dir = dest_dir
+ venv = os.path.join(parent_dir, 'venv')
+
if update_requirements:
if not requirements_dir:
error_out('requirements repo must be cloned before '
'updating from global requirements.')
- _git_update_requirements(repo_dir, requirements_dir)
+ _git_update_requirements(venv, repo_dir, requirements_dir)
juju_log('Installing git repo from dir: {}'.format(repo_dir))
if http_proxy:
- pip_install(repo_dir, proxy=http_proxy,
- venv=os.path.join(parent_dir, 'venv'))
+ pip_install(repo_dir, proxy=http_proxy, venv=venv)
else:
- pip_install(repo_dir,
- venv=os.path.join(parent_dir, 'venv'))
+ pip_install(repo_dir, venv=venv)
return repo_dir
-def _git_update_requirements(package_dir, reqs_dir):
+def _git_update_requirements(venv, package_dir, reqs_dir):
"""
Update from global requirements.
@@ -636,12 +647,14 @@ def _git_update_requirements(package_dir, reqs_dir):
"""
orig_dir = os.getcwd()
os.chdir(reqs_dir)
- cmd = ['python', 'update.py', package_dir]
+ python = os.path.join(venv, 'bin/python')
+ cmd = [python, 'update.py', package_dir]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
package = os.path.basename(package_dir)
- error_out("Error updating {} from global-requirements.txt".format(package))
+ error_out("Error updating {} from "
+ "global-requirements.txt".format(package))
os.chdir(orig_dir)
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 411d0f2d..117429fd 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -364,11 +364,16 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
relation_settings = relation_settings if relation_settings else {}
relation_cmd_line = ['relation-set']
accepts_file = "--file" in subprocess.check_output(
- relation_cmd_line + ["--help"])
+ relation_cmd_line + ["--help"], universal_newlines=True)
if relation_id is not None:
relation_cmd_line.extend(('-r', relation_id))
settings = relation_settings.copy()
settings.update(kwargs)
+ for key, value in settings.items():
+ # Force value to be a string: it always should, but some call
+ # sites pass in things like dicts or numbers.
+ if value is not None:
+ settings[key] = "{}".format(value)
if accepts_file:
# --file was introduced in Juju 1.23.2. Use it by default if
# available, since otherwise we'll break if the relation data is
@@ -390,6 +395,17 @@ def relation_set(relation_id=None, relation_settings=None, **kwargs):
flush(local_unit())
+def relation_clear(r_id=None):
+ ''' Clears any relation data already set on relation r_id '''
+ settings = relation_get(rid=r_id,
+ unit=local_unit())
+ for setting in settings:
+ if setting not in ['public-address', 'private-address']:
+ settings[setting] = None
+ relation_set(relation_id=r_id,
+ **settings)
+
+
@cached
def relation_ids(reltype=None):
"""A list of relation_ids"""
@@ -681,3 +697,48 @@ def status_get():
return 'unknown'
else:
raise
+
+
+def translate_exc(from_exc, to_exc):
+ def inner_translate_exc1(f):
+ def inner_translate_exc2(*args, **kwargs):
+ try:
+ return f(*args, **kwargs)
+ except from_exc:
+ raise to_exc
+
+ return inner_translate_exc2
+
+ return inner_translate_exc1
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def is_leader():
+ """Does the current unit hold the juju leadership
+
+ Uses juju to determine whether the current unit is the leader of its peers
+ """
+ cmd = ['is-leader', '--format=json']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_get(attribute=None):
+ """Juju leader get value(s)"""
+ cmd = ['leader-get', '--format=json'] + [attribute or '-']
+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
+
+
+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
+def leader_set(settings=None, **kwargs):
+ """Juju leader set value(s)"""
+ log("Juju leader-set '%s'" % (settings), level=DEBUG)
+ cmd = ['leader-set']
+ settings = settings or {}
+ settings.update(kwargs)
+ for k, v in settings.iteritems():
+ if v is None:
+ cmd.append('{}='.format(k))
+ else:
+ cmd.append('{}={}'.format(k, v))
+ subprocess.check_call(cmd)
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 0d2ab4b4..901a4cfe 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -24,6 +24,7 @@
import os
import re
import pwd
+import glob
import grp
import random
import string
@@ -269,6 +270,21 @@ def file_hash(path, hash_type='md5'):
return None
+def path_hash(path):
+ """
+ Generate a hash checksum of all files matching 'path'. Standard wildcards
+ like '*' and '?' are supported, see documentation for the 'glob' module for
+ more information.
+
+ :return: dict: A { filename: hash } dictionary for all matched files.
+ Empty if none found.
+ """
+ return {
+ filename: file_hash(filename)
+ for filename in glob.iglob(path)
+ }
+
+
def check_hash(path, checksum, hash_type='md5'):
"""
Validate a file using a cryptographic checksum.
@@ -296,23 +312,25 @@ def restart_on_change(restart_map, stopstart=False):
@restart_on_change({
'/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+ '/etc/apache/sites-enabled/*': [ 'apache2' ]
})
- def ceph_client_changed():
+ def config_changed():
pass # your code here
In this example, the cinder-api and cinder-volume services
would be restarted if /etc/ceph/ceph.conf is changed by the
- ceph_client_changed function.
+ ceph_client_changed function. The apache2 service would be
+ restarted if any file matching the pattern got changed, created
+ or removed. Standard wildcards are supported, see documentation
+ for the 'glob' module for more information.
"""
def wrap(f):
def wrapped_f(*args, **kwargs):
- checksums = {}
- for path in restart_map:
- checksums[path] = file_hash(path)
+ checksums = {path: path_hash(path) for path in restart_map}
f(*args, **kwargs)
restarts = []
for path in restart_map:
- if checksums[path] != file_hash(path):
+ if path_hash(path) != checksums[path]:
restarts += restart_map[path]
services_list = list(OrderedDict.fromkeys(restarts))
if not stopstart:
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
index 2ff53381..98d344e1 100644
--- a/hooks/charmhelpers/core/services/base.py
+++ b/hooks/charmhelpers/core/services/base.py
@@ -15,8 +15,8 @@
# along with charm-helpers. If not, see .
import os
-import re
import json
+from inspect import getargspec
from collections import Iterable, OrderedDict
from charmhelpers.core import host
@@ -132,8 +132,8 @@ class ServiceManager(object):
if hook_name == 'stop':
self.stop_services()
else:
- self.provide_data()
self.reconfigure_services()
+ self.provide_data()
cfg = hookenv.config()
if cfg.implicit_save:
cfg.save()
@@ -145,15 +145,36 @@ class ServiceManager(object):
A provider must have a `name` attribute, which indicates which relation
to set data on, and a `provide_data()` method, which returns a dict of
data to set.
+
+ The `provide_data()` method can optionally accept two parameters:
+
+ * ``remote_service`` The name of the remote service that the data will
+ be provided to. The `provide_data()` method will be called once
+ for each connected service (not unit). This allows the method to
+ tailor its data to the given service.
+ * ``service_ready`` Whether or not the service definition had all of
+ its requirements met, and thus the ``data_ready`` callbacks run.
+
+ Note that the ``provided_data`` methods are now called **after** the
+ ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
+ a chance to generate any data necessary for the providing to the remote
+ services.
"""
- hook_name = hookenv.hook_name()
- for service in self.services.values():
+ for service_name, service in self.services.items():
+ service_ready = self.is_ready(service_name)
for provider in service.get('provided_data', []):
- if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
- data = provider.provide_data()
- _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
- if _ready:
- hookenv.relation_set(None, data)
+ for relid in hookenv.relation_ids(provider.name):
+ units = hookenv.related_units(relid)
+ if not units:
+ continue
+ remote_service = units[0].split('/')[0]
+ argspec = getargspec(provider.provide_data)
+ if len(argspec.args) > 1:
+ data = provider.provide_data(remote_service, service_ready)
+ else:
+ data = provider.provide_data()
+ if data:
+ hookenv.relation_set(relid, data)
def reconfigure_services(self, *service_names):
"""
diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py
index f61c2e8b..e8c4a274 100644
--- a/tests/charmhelpers/contrib/amulet/utils.py
+++ b/tests/charmhelpers/contrib/amulet/utils.py
@@ -15,13 +15,15 @@
# along with charm-helpers. If not, see .
import ConfigParser
+import distro_info
import io
import logging
+import os
import re
+import six
import sys
import time
-
-import six
+import urlparse
class AmuletUtils(object):
@@ -33,6 +35,7 @@ class AmuletUtils(object):
def __init__(self, log_level=logging.ERROR):
self.log = self.get_logger(level=log_level)
+ self.ubuntu_releases = self.get_ubuntu_releases()
def get_logger(self, name="amulet-logger", level=logging.DEBUG):
"""Get a logger object that will log to stdout."""
@@ -70,12 +73,44 @@ class AmuletUtils(object):
else:
return False
- def validate_services(self, commands):
- """Validate services.
+ def get_ubuntu_release_from_sentry(self, sentry_unit):
+ """Get Ubuntu release codename from sentry unit.
- Verify the specified services are running on the corresponding
+ :param sentry_unit: amulet sentry/service unit pointer
+ :returns: list of strings - release codename, failure message
+ """
+ msg = None
+ cmd = 'lsb_release -cs'
+ release, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} lsb_release: {}'.format(
+ sentry_unit.info['unit_name'], release))
+ else:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, release, code))
+ if release not in self.ubuntu_releases:
+ msg = ("Release ({}) not found in Ubuntu releases "
+ "({})".format(release, self.ubuntu_releases))
+ return release, msg
+
+ def validate_services(self, commands):
+ """Validate that lists of commands succeed on service units. Can be
+ used to verify system services are running on the corresponding
service units.
- """
+
+ :param commands: dict with sentry keys and arbitrary command list vals
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # /!\ DEPRECATION WARNING (beisner):
+ # New and existing tests should be rewritten to use
+ # validate_services_by_name() as it is aware of init systems.
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'validate_services_by_name instead of validate_services '
+ 'due to init system differences.')
+
for k, v in six.iteritems(commands):
for cmd in v:
output, code = k.run(cmd)
@@ -86,6 +121,41 @@ class AmuletUtils(object):
return "command `{}` returned {}".format(cmd, str(code))
return None
+ def validate_services_by_name(self, sentry_services):
+ """Validate system service status by service name, automatically
+ detecting init system based on Ubuntu release codename.
+
+ :param sentry_services: dict with sentry keys and svc list values
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # Point at which systemd became a thing
+ systemd_switch = self.ubuntu_releases.index('vivid')
+
+ for sentry_unit, services_list in six.iteritems(sentry_services):
+ # Get lsb_release codename from unit
+ release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+ if ret:
+ return ret
+
+ for service_name in services_list:
+ if (self.ubuntu_releases.index(release) >= systemd_switch or
+ service_name == "rabbitmq-server"):
+ # init is systemd
+ cmd = 'sudo service {} status'.format(service_name)
+ elif self.ubuntu_releases.index(release) < systemd_switch:
+ # init is upstart
+ cmd = 'sudo status {}'.format(service_name)
+
+ output, code = sentry_unit.run(cmd)
+ self.log.debug('{} `{}` returned '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ if code != 0:
+ return "command `{}` returned {}".format(cmd, str(code))
+ return None
+
def _get_config(self, unit, filename):
"""Get a ConfigParser object for parsing a unit's config file."""
file_contents = unit.file_contents(filename)
@@ -104,6 +174,9 @@ class AmuletUtils(object):
Verify that the specified section of the config file contains
the expected option key:value pairs.
"""
+ self.log.debug('Validating config file data ({} in {} on {})'
+ '...'.format(section, config_file,
+ sentry_unit.info['unit_name']))
config = self._get_config(sentry_unit, config_file)
if section != 'DEFAULT' and not config.has_section(section):
@@ -321,3 +394,15 @@ class AmuletUtils(object):
def endpoint_error(self, name, data):
return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+ def get_ubuntu_releases(self):
+ """Return a list of all Ubuntu releases in order of release."""
+ _d = distro_info.UbuntuDistroInfo()
+ _release_list = _d.all
+ self.log.debug('Ubuntu release list: {}'.format(_release_list))
+ return _release_list
+
+ def file_to_url(self, file_rel_path):
+ """Convert a relative file path to a file URL."""
+ _abs_path = os.path.abspath(file_rel_path)
+ return urlparse.urlparse(_abs_path, scheme='file').geturl()
diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
index 461a702f..c664c9d0 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py
@@ -110,7 +110,8 @@ class OpenStackAmuletDeployment(AmuletDeployment):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
- self.trusty_kilo, self.vivid_kilo) = range(10)
+ self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
+ self.wily_liberty) = range(12)
releases = {
('precise', None): self.precise_essex,
@@ -121,8 +122,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+ ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('utopic', None): self.utopic_juno,
- ('vivid', None): self.vivid_kilo}
+ ('vivid', None): self.vivid_kilo,
+ ('wily', None): self.wily_liberty}
return releases[(self.series, self.openstack)]
def _get_openstack_release_string(self):
@@ -138,6 +141,7 @@ class OpenStackAmuletDeployment(AmuletDeployment):
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
+ ('wily', 'liberty'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]
diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py
index 9c3d918a..576bf0b5 100644
--- a/tests/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py
@@ -16,15 +16,15 @@
import logging
import os
+import six
import time
import urllib
import glanceclient.v1.client as glance_client
+import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
import novaclient.v1_1.client as nova_client
-import six
-
from charmhelpers.contrib.amulet.utils import (
AmuletUtils
)
@@ -37,7 +37,7 @@ class OpenStackAmuletUtils(AmuletUtils):
"""OpenStack amulet utilities.
This class inherits from AmuletUtils and has additional support
- that is specifically for use by OpenStack charms.
+ that is specifically for use by OpenStack charm tests.
"""
def __init__(self, log_level=ERROR):
@@ -51,6 +51,8 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate actual endpoint data vs expected endpoint data. The ports
are used to find the matching endpoint.
"""
+ self.log.debug('Validating endpoint data...')
+ self.log.debug('actual: {}'.format(repr(endpoints)))
found = False
for ep in endpoints:
self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -77,6 +79,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual service catalog endpoints vs a list of
expected service catalog endpoints.
"""
+ self.log.debug('Validating service catalog endpoint data...')
self.log.debug('actual: {}'.format(repr(actual)))
for k, v in six.iteritems(expected):
if k in actual:
@@ -93,6 +96,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual tenant data vs list of expected tenant
data.
"""
+ self.log.debug('Validating tenant data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -114,6 +118,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual role data vs a list of expected role
data.
"""
+ self.log.debug('Validating role data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -134,6 +139,7 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual user data vs a list of expected user
data.
"""
+ self.log.debug('Validating user data...')
self.log.debug('actual: {}'.format(repr(actual)))
for e in expected:
found = False
@@ -155,17 +161,20 @@ class OpenStackAmuletUtils(AmuletUtils):
Validate a list of actual flavors vs a list of expected flavors.
"""
+ self.log.debug('Validating flavor data...')
self.log.debug('actual: {}'.format(repr(actual)))
act = [a.name for a in actual]
return self._validate_list_data(expected, act)
def tenant_exists(self, keystone, tenant):
"""Return True if tenant exists."""
+ self.log.debug('Checking if tenant exists ({})...'.format(tenant))
return tenant in [t.name for t in keystone.tenants.list()]
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant):
"""Authenticates admin user with the keystone admin endpoint."""
+ self.log.debug('Authenticating keystone admin...')
unit = keystone_sentry
service_ip = unit.relation('shared-db',
'mysql:shared-db')['private-address']
@@ -175,6 +184,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
+ self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return keystone_client.Client(username=user, password=password,
@@ -182,12 +192,21 @@ class OpenStackAmuletUtils(AmuletUtils):
def authenticate_glance_admin(self, keystone):
"""Authenticates admin user with glance."""
+ self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
+ def authenticate_heat_admin(self, keystone):
+ """Authenticates the admin user with heat."""
+ self.log.debug('Authenticating heat admin...')
+ ep = keystone.service_catalog.url_for(service_type='orchestration',
+ endpoint_type='publicURL')
+ return heat_client.Client(endpoint=ep, token=keystone.auth_token)
+
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
+ self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(username=user, api_key=password,
@@ -195,6 +214,7 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance."""
+ self.log.debug('Creating glance image ({})...'.format(image_name))
http_proxy = os.getenv('AMULET_HTTP_PROXY')
self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
if http_proxy:
@@ -235,6 +255,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_image(self, glance, image):
"""Delete the specified image."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_image.')
+ self.log.debug('Deleting glance image ({})...'.format(image))
num_before = len(list(glance.images.list()))
glance.images.delete(image)
@@ -254,6 +279,8 @@ class OpenStackAmuletUtils(AmuletUtils):
def create_instance(self, nova, image_name, instance_name, flavor):
"""Create the specified instance."""
+ self.log.debug('Creating instance '
+ '({}|{}|{})'.format(instance_name, image_name, flavor))
image = nova.images.find(name=image_name)
flavor = nova.flavors.find(name=flavor)
instance = nova.servers.create(name=instance_name, image=image,
@@ -276,6 +303,11 @@ class OpenStackAmuletUtils(AmuletUtils):
def delete_instance(self, nova, instance):
"""Delete the specified instance."""
+
+ # /!\ DEPRECATION WARNING
+ self.log.warn('/!\\ DEPRECATION WARNING: use '
+ 'delete_resource instead of delete_instance.')
+ self.log.debug('Deleting instance ({})...'.format(instance))
num_before = len(list(nova.servers.list()))
nova.servers.delete(instance)
@@ -292,3 +324,90 @@ class OpenStackAmuletUtils(AmuletUtils):
return False
return True
+
+ def create_or_get_keypair(self, nova, keypair_name="testkey"):
+ """Create a new keypair, or return pointer if it already exists."""
+ try:
+ _keypair = nova.keypairs.get(keypair_name)
+ self.log.debug('Keypair ({}) already exists, '
+ 'using it.'.format(keypair_name))
+ return _keypair
+ except:
+ self.log.debug('Keypair ({}) does not exist, '
+ 'creating it.'.format(keypair_name))
+
+ _keypair = nova.keypairs.create(name=keypair_name)
+ return _keypair
+
+ def delete_resource(self, resource, resource_id,
+ msg="resource", max_wait=120):
+ """Delete one openstack resource, such as one instance, keypair,
+ image, volume, stack, etc., and confirm deletion within max wait time.
+
+ :param resource: pointer to os resource type, ex:glance_client.images
+ :param resource_id: unique name or id for the openstack resource
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, otherwise False
+ """
+ num_before = len(list(resource.list()))
+ resource.delete(resource_id)
+
+ tries = 0
+ num_after = len(list(resource.list()))
+ while num_after != (num_before - 1) and tries < (max_wait / 4):
+ self.log.debug('{} delete check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ num_before,
+ num_after,
+ resource_id))
+ time.sleep(4)
+ num_after = len(list(resource.list()))
+ tries += 1
+
+ self.log.debug('{}: expected, actual count = {}, '
+ '{}'.format(msg, num_before - 1, num_after))
+
+ if num_after == (num_before - 1):
+ return True
+ else:
+ self.log.error('{} delete timed out'.format(msg))
+ return False
+
+ def resource_reaches_status(self, resource, resource_id,
+ expected_stat='available',
+ msg='resource', max_wait=120):
+ """Wait for an openstack resources status to reach an
+ expected status within a specified time. Useful to confirm that
+ nova instances, cinder vols, snapshots, glance images, heat stacks
+ and other resources eventually reach the expected status.
+
+ :param resource: pointer to os resource type, ex: heat_client.stacks
+ :param resource_id: unique id for the openstack resource
+ :param expected_stat: status to expect resource to reach
+ :param msg: text to identify purpose in logging
+ :param max_wait: maximum wait time in seconds
+ :returns: True if successful, False if status is not reached
+ """
+
+ tries = 0
+ resource_stat = resource.get(resource_id).status
+ while resource_stat != expected_stat and tries < (max_wait / 4):
+ self.log.debug('{} status check: '
+ '{} [{}:{}] {}'.format(msg, tries,
+ resource_stat,
+ expected_stat,
+ resource_id))
+ time.sleep(4)
+ resource_stat = resource.get(resource_id).status
+ tries += 1
+
+ self.log.debug('{}: expected, actual status = {}, '
+ '{}'.format(msg, resource_stat, expected_stat))
+
+ if resource_stat == expected_stat:
+ return True
+ else:
+ self.log.debug('{} never reached expected status: '
+ '{}'.format(resource_id, expected_stat))
+ return False