openstack/charm-nova-compute
Code Issues Proposed changes

Make the cloud-compute relation net spaces aware

Browse Source 
When nova compute is deployed with multiple network interfaces
configured. It may be necessary to specify which interface to use for
the cloud-compute relation.

Do not use unit_get('private-address') which gives unpredictable
results. Instead, leverage network-get for network spaces aware address
selection.

Charm helpers sync to pull in generalized get_relation_ip() used in
other charms which checks network-get after checking for all of the edge
cases including IPv6 and config overrides. Use get_relation_ip() for
address selection.

This allows for specifying a network space in a bundle when using MAAS
2.x. This guarantees the charm will select the correct interface and IP
for the network space.

nova-compute:
  bindings:
    cloud-compute: internal-space

Closes-bug: #1670866

Change-Id: Ib43a7a52acf5a07b68dea808082da0ba6eb237c1
This commit is contained in:
David Ames 2017-03-07 08:33:13 -08:00
parent 01795a4ff4
commit 3f3a159704
19 changed files with 630 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hooks/charmhelpers/contrib/charmsupport/nrpe.py
View File

@ -338,13 +338,14 @@ def get_nagios_unit_name(relation_name='nrpe-external-master'):
return unit
def add_init_service_checks(nrpe, services, unit_name):
def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
"""
Add checks for each service in list
:param NRPE nrpe: NRPE object to add check to
:param list services: List of services to check
:param str unit_name: Unit name to use in check description
:param bool immediate_check: For sysv init, run the service check immediately
"""
for svc in services:
# Don't add a check for these services from neutron-gateway
@ -368,21 +369,29 @@ def add_init_service_checks(nrpe, services, unit_name):
)
elif os.path.exists(sysv_init):
cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
cron_file = ('*/5 * * * * root '
'/usr/local/lib/nagios/plugins/check_exit_status.pl '
'-s /etc/init.d/%s status > '
'/var/lib/nagios/service-check-%s.txt\n' % (svc,
svc)
)
checkpath = '/var/lib/nagios/service-check-%s.txt' % svc
croncmd = (
'/usr/local/lib/nagios/plugins/check_exit_status.pl '
'-s /etc/init.d/%s status' % svc
)
cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
f = open(cronpath, 'w')
f.write(cron_file)
f.close()
nrpe.add_check(
shortname=svc,
description='process check {%s}' % unit_name,
check_cmd='check_status_file.py -f '
'/var/lib/nagios/service-check-%s.txt' % svc,
description='service check {%s}' % unit_name,
check_cmd='check_status_file.py -f %s' % checkpath,
)
if immediate_check:
f = open(checkpath, 'w')
subprocess.call(
croncmd.split(),
stdout=f,
stderr=subprocess.STDOUT
)
f.close()
os.chmod(checkpath, 0o644)
def copy_nrpe_checks():

6
hooks/charmhelpers/contrib/hardening/templating.py
View File

@ -13,6 +13,7 @@
# limitations under the License.
import os
import six
from charmhelpers.core.hookenv import (
log,
@ -26,7 +27,10 @@ except ImportError:
from charmhelpers.fetch import apt_install
from charmhelpers.fetch import apt_update
apt_update(fatal=True)
apt_install('python-jinja2', fatal=True)
if six.PY2:
apt_install('python-jinja2', fatal=True)
else:
apt_install('python3-jinja2', fatal=True)
from jinja2 import FileSystemLoader, Environment

65
hooks/charmhelpers/contrib/network/ip.py
View File

@ -20,25 +20,37 @@ import socket
from functools import partial
from charmhelpers.core.hookenv import unit_get
from charmhelpers.fetch import apt_install, apt_update
from charmhelpers.core.hookenv import (
config,
log,
network_get_primary_address,
unit_get,
WARNING,
)
from charmhelpers.core.host import (
lsb_release,
)
try:
import netifaces
except ImportError:
apt_update(fatal=True)
apt_install('python-netifaces', fatal=True)
if six.PY2:
apt_install('python-netifaces', fatal=True)
else:
apt_install('python3-netifaces', fatal=True)
import netifaces
try:
import netaddr
except ImportError:
apt_update(fatal=True)
apt_install('python-netaddr', fatal=True)
if six.PY2:
apt_install('python-netaddr', fatal=True)
else:
apt_install('python3-netaddr', fatal=True)
import netaddr
@ -414,7 +426,10 @@ def ns_query(address):
try:
import dns.resolver
except ImportError:
apt_install('python-dnspython', fatal=True)
if six.PY2:
apt_install('python-dnspython', fatal=True)
else:
apt_install('python3-dnspython', fatal=True)
import dns.resolver
if isinstance(address, dns.name.Name):
@ -462,7 +477,10 @@ def get_hostname(address, fqdn=True):
try:
import dns.reversename
except ImportError:
apt_install("python-dnspython", fatal=True)
if six.PY2:
apt_install("python-dnspython", fatal=True)
else:
apt_install("python3-dnspython", fatal=True)
import dns.reversename
rev = dns.reversename.from_address(address)
@ -499,3 +517,40 @@ def port_has_listener(address, port):
cmd = ['nc', '-z', address, str(port)]
result = subprocess.call(cmd)
return not(bool(result))
def assert_charm_supports_ipv6():
"""Check whether we are able to support charms ipv6."""
if lsb_release()['DISTRIB_CODENAME'].lower() < "trusty":
raise Exception("IPv6 is not supported in the charms for Ubuntu "
"versions less than Trusty 14.04")
def get_relation_ip(interface, config_override=None):
"""Return this unit's IP for the given relation.
Allow for an arbitrary interface to use with network-get to select an IP.
Handle all address selection options including configuration parameter
override and IPv6.
Usage: get_relation_ip('amqp', config_override='access-network')
@param interface: string name of the relation.
@param config_override: string name of the config option for network
override. Supports legacy network override configuration parameters.
@raises Exception if prefer-ipv6 is configured but IPv6 unsupported.
@returns IPv6 or IPv4 address
"""
fallback = get_host_ip(unit_get('private-address'))
if config('prefer-ipv6'):
assert_charm_supports_ipv6()
return get_ipv6_addr()[0]
elif config_override and config(config_override):
return get_address_in_network(config(config_override),
fallback)
else:
try:
return network_get_primary_address(interface)
except NotImplementedError:
return fallback

67
hooks/charmhelpers/contrib/network/ovs/__init__.py
View File

@ -15,13 +15,30 @@
''' Helpers for interacting with OpenvSwitch '''
import subprocess
import os
import six
from charmhelpers.fetch import apt_install
from charmhelpers.core.hookenv import (
log, WARNING
log, WARNING, INFO, DEBUG
)
from charmhelpers.core.host import (
service
)
BRIDGE_TEMPLATE = """\
# This veth pair is required when neutron data-port is mapped to an existing linux bridge. lp:1635067
auto {linuxbridge_port}
iface {linuxbridge_port} inet manual
pre-up ip link add name {linuxbridge_port} type veth peer name {ovsbridge_port}
pre-up ip link set {ovsbridge_port} master {bridge}
pre-up ip link set {ovsbridge_port} up
up ip link set {linuxbridge_port} up
down ip link del {linuxbridge_port}
"""
def add_bridge(name, datapath_type=None):
''' Add the named bridge to openvswitch '''
@ -60,6 +77,54 @@ def del_bridge_port(name, port):
subprocess.check_call(["ip", "link", "set", port, "promisc", "off"])
def add_ovsbridge_linuxbridge(name, bridge):
''' Add linux bridge to the named openvswitch bridge
:param name: Name of ovs bridge to be added to Linux bridge
:param bridge: Name of Linux bridge to be added to ovs bridge
:returns: True if veth is added between ovs bridge and linux bridge,
False otherwise'''
try:
import netifaces
except ImportError:
if six.PY2:
apt_install('python-netifaces', fatal=True)
else:
apt_install('python3-netifaces', fatal=True)
import netifaces
ovsbridge_port = "veth-" + name
linuxbridge_port = "veth-" + bridge
log('Adding linuxbridge {} to ovsbridge {}'.format(bridge, name),
level=INFO)
interfaces = netifaces.interfaces()
for interface in interfaces:
if interface == ovsbridge_port or interface == linuxbridge_port:
log('Interface {} already exists'.format(interface), level=INFO)
return
with open('/etc/network/interfaces.d/{}.cfg'.format(
linuxbridge_port), 'w') as config:
config.write(BRIDGE_TEMPLATE.format(linuxbridge_port=linuxbridge_port,
ovsbridge_port=ovsbridge_port,
bridge=bridge))
subprocess.check_call(["ifup", linuxbridge_port])
add_bridge_port(name, linuxbridge_port)
def is_linuxbridge_interface(port):
''' Check if the interface is a linuxbridge bridge
:param port: Name of an interface to check whether it is a Linux bridge
:returns: True if port is a Linux bridge'''
if os.path.exists('/sys/class/net/' + port + '/bridge'):
log('Interface {} is a Linux bridge'.format(port), level=DEBUG)
return True
else:
log('Interface {} is not a Linux bridge'.format(port), level=DEBUG)
return False
def set_manager(manager):
''' Set the controller for the local openvswitch '''
log('Setting manager for local ovs to {}'.format(manager))

12
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -32,6 +32,7 @@ from keystoneclient.v3 import client as keystone_client_v3
from novaclient import exceptions
import novaclient.client as nova_client
import novaclient
import pika
import swiftclient
@ -434,9 +435,14 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, api_key=password,
project_id=tenant, auth_url=ep)
if novaclient.__version__[0] >= "7":
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, password=password,
project_name=tenant, auth_url=ep)
else:
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, api_key=password,
project_id=tenant, auth_url=ep)
def authenticate_swift_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with swift api."""

39
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -100,7 +100,10 @@ from charmhelpers.core.unitdata import kv
try:
import psutil
except ImportError:
apt_install('python-psutil', fatal=True)
if six.PY2:
apt_install('python-psutil', fatal=True)
else:
apt_install('python3-psutil', fatal=True)
import psutil
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@ -392,16 +395,20 @@ class AMQPContext(OSContextGenerator):
for rid in relation_ids(self.rel_name):
ha_vip_only = False
self.related = True
transport_hosts = None
rabbitmq_port = '5672'
for unit in related_units(rid):
if relation_get('clustered', rid=rid, unit=unit):
ctxt['clustered'] = True
vip = relation_get('vip', rid=rid, unit=unit)
vip = format_ipv6_addr(vip) or vip
ctxt['rabbitmq_host'] = vip
transport_hosts = [vip]
else:
host = relation_get('private-address', rid=rid, unit=unit)
host = format_ipv6_addr(host) or host
ctxt['rabbitmq_host'] = host
transport_hosts = [host]
ctxt.update({
'rabbitmq_user': username,
@ -413,6 +420,7 @@ class AMQPContext(OSContextGenerator):
ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
if ssl_port:
ctxt['rabbit_ssl_port'] = ssl_port
rabbitmq_port = ssl_port
ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
if ssl_ca:
@ -450,6 +458,20 @@ class AMQPContext(OSContextGenerator):
rabbitmq_hosts.append(host)
ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
transport_hosts = rabbitmq_hosts
if transport_hosts:
transport_url_hosts = ''
for host in transport_hosts:
if transport_url_hosts:
format_string = ",{}:{}@{}:{}"
else:
format_string = "{}:{}@{}:{}"
transport_url_hosts += format_string.format(
ctxt['rabbitmq_user'], ctxt['rabbitmq_password'],
host, rabbitmq_port)
ctxt['transport_url'] = "rabbit://{}/{}".format(
transport_url_hosts, vhost)
oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
if oslo_messaging_flags:
@ -481,13 +503,16 @@ class CephContext(OSContextGenerator):
ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
if not ctxt.get('key'):
ctxt['key'] = relation_get('key', rid=rid, unit=unit)
ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
ceph_addrs = relation_get('ceph-public-address', rid=rid,
unit=unit)
if ceph_addrs:
for addr in ceph_addrs.split(' '):
mon_hosts.append(format_ipv6_addr(addr) or addr)
else:
priv_addr = relation_get('private-address', rid=rid,
unit=unit)
unit_priv_addr = relation_get('private-address', rid=rid,
unit=unit)
ceph_addr = ceph_pub_addr or unit_priv_addr
ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
mon_hosts.append(ceph_addr)
mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr)
ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))

11
hooks/charmhelpers/contrib/openstack/ha/utils.py
View File

@ -126,3 +126,14 @@ def assert_charm_supports_dns_ha():
status_set('blocked', msg)
raise DNSHAException(msg)
return True
def expect_ha():
""" Determine if the unit expects to be in HA
Check for VIP or dns-ha settings which indicate the unit should expect to
be related to hacluster.
@returns boolean
"""
return config('vip') or config('dns-ha')

178
hooks/charmhelpers/contrib/openstack/keystone.py Normal file
View File

@ -0,0 +1,178 @@
#!/usr/bin/python
#
# Copyright 2017 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import six
from charmhelpers.fetch import apt_install
from charmhelpers.contrib.openstack.context import IdentityServiceContext
from charmhelpers.core.hookenv import (
log,
ERROR,
)
def get_api_suffix(api_version):
"""Return the formatted api suffix for the given version
@param api_version: version of the keystone endpoint
@returns the api suffix formatted according to the given api
version
"""
return 'v2.0' if api_version in (2, "2.0") else 'v3'
def format_endpoint(schema, addr, port, api_version):
"""Return a formatted keystone endpoint
@param schema: http or https
@param addr: ipv4/ipv6 host of the keystone service
@param port: port of the keystone service
@param api_version: 2 or 3
@returns a fully formatted keystone endpoint
"""
return '{}://{}:{}/{}/'.format(schema, addr, port,
get_api_suffix(api_version))
def get_keystone_manager(endpoint, api_version, **kwargs):
"""Return a keystonemanager for the correct API version
@param endpoint: the keystone endpoint to point client at
@param api_version: version of the keystone api the client should use
@param kwargs: token or username/tenant/password information
@returns keystonemanager class used for interrogating keystone
"""
if api_version == 2:
return KeystoneManager2(endpoint, **kwargs)
if api_version == 3:
return KeystoneManager3(endpoint, **kwargs)
raise ValueError('No manager found for api version {}'.format(api_version))
def get_keystone_manager_from_identity_service_context():
"""Return a keystonmanager generated from a
instance of charmhelpers.contrib.openstack.context.IdentityServiceContext
@returns keystonamenager instance
"""
context = IdentityServiceContext()()
if not context:
msg = "Identity service context cannot be generated"
log(msg, level=ERROR)
raise ValueError(msg)
endpoint = format_endpoint(context['service_protocol'],
context['service_host'],
context['service_port'],
context['api_version'])
if context['api_version'] in (2, "2.0"):
api_version = 2
else:
api_version = 3
return get_keystone_manager(endpoint, api_version,
username=context['admin_user'],
password=context['admin_password'],
tenant_name=context['admin_tenant_name'])
class KeystoneManager(object):
def resolve_service_id(self, service_name=None, service_type=None):
"""Find the service_id of a given service"""
services = [s._info for s in self.api.services.list()]
service_name = service_name.lower()
for s in services:
name = s['name'].lower()
if service_type and service_name:
if (service_name == name and service_type == s['type']):
return s['id']
elif service_name and service_name == name:
return s['id']
elif service_type and service_type == s['type']:
return s['id']
return None
def service_exists(self, service_name=None, service_type=None):
"""Determine if the given service exists on the service list"""
return self.resolve_service_id(service_name, service_type) is not None
class KeystoneManager2(KeystoneManager):
def __init__(self, endpoint, **kwargs):
try:
from keystoneclient.v2_0 import client
from keystoneclient.auth.identity import v2
from keystoneclient import session
except ImportError:
if six.PY2:
apt_install(["python-keystoneclient"], fatal=True)
else:
apt_install(["python3-keystoneclient"], fatal=True)
from keystoneclient.v2_0 import client
from keystoneclient.auth.identity import v2
from keystoneclient import session
self.api_version = 2
token = kwargs.get("token", None)
if token:
api = client.Client(endpoint=endpoint, token=token)
else:
auth = v2.Password(username=kwargs.get("username"),
password=kwargs.get("password"),
tenant_name=kwargs.get("tenant_name"),
auth_url=endpoint)
sess = session.Session(auth=auth)
api = client.Client(session=sess)
self.api = api
class KeystoneManager3(KeystoneManager):
def __init__(self, endpoint, **kwargs):
try:
from keystoneclient.v3 import client
from keystoneclient.auth import token_endpoint
from keystoneclient import session
from keystoneclient.auth.identity import v3
except ImportError:
if six.PY2:
apt_install(["python-keystoneclient"], fatal=True)
else:
apt_install(["python3-keystoneclient"], fatal=True)
from keystoneclient.v3 import client
from keystoneclient.auth import token_endpoint
from keystoneclient import session
from keystoneclient.auth.identity import v3
self.api_version = 3
token = kwargs.get("token", None)
if token:
auth = token_endpoint.Token(endpoint=endpoint,
token=token)
sess = session.Session(auth=auth)
else:
auth = v3.Password(auth_url=endpoint,
user_id=kwargs.get("username"),
password=kwargs.get("password"),
project_id=kwargs.get("tenant_name"))
sess = session.Session(auth=auth)
self.api = client.Client(session=sess)

10
hooks/charmhelpers/contrib/openstack/templating.py
View File

@ -28,7 +28,10 @@ try:
from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
except ImportError:
apt_update(fatal=True)
apt_install('python-jinja2', fatal=True)
if six.PY2:
apt_install('python-jinja2', fatal=True)
else:
apt_install('python3-jinja2', fatal=True)
from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
@ -207,7 +210,10 @@ class OSConfigRenderer(object):
# if this code is running, the object is created pre-install hook.
# jinja2 shouldn't get touched until the module is reloaded on next
# hook execution, with proper jinja2 bits successfully imported.
apt_install('python-jinja2')
if six.PY2:
apt_install('python-jinja2')
else:
apt_install('python3-jinja2')
def register(self, config_file, contexts):
"""

2
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([
('newton',
['2.8.0', '2.9.0', '2.10.0']),
('ocata',
['2.11.0', '2.12.0']),
['2.11.0', '2.12.0', '2.13.0']),
])
# >= Liberty version->codename mapping

11
hooks/charmhelpers/contrib/python/packages.py
View File

@ -16,6 +16,7 @@
# limitations under the License.
import os
import six
import subprocess
import sys
@ -39,7 +40,10 @@ def pip_execute(*args, **kwargs):
from pip import main as _pip_execute
except ImportError:
apt_update()
apt_install('python-pip')
if six.PY2:
apt_install('python-pip')
else:
apt_install('python3-pip')
from pip import main as _pip_execute
_pip_execute(*args, **kwargs)
finally:
@ -136,7 +140,10 @@ def pip_list():
def pip_create_virtualenv(path=None):
"""Create an isolated Python environment."""
apt_install('python-virtualenv')
if six.PY2:
apt_install('python-virtualenv')
else:
apt_install('python3-virtualenv')
if path:
venv_path = path

122
hooks/charmhelpers/fetch/snap.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2014-2017 Canonical Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Charm helpers snap for classic charms.
If writing reactive charms, use the snap layer:
https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
"""
import subprocess
from os import environ
from time import sleep
from charmhelpers.core.hookenv import log
__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
class CouldNotAcquireLockException(Exception):
pass
def _snap_exec(commands):
"""
Execute snap commands.
:param commands: List commands
:return: Integer exit code
"""
assert type(commands) == list
retry_count = 0
return_code = None
while return_code is None or return_code == SNAP_NO_LOCK:
try:
return_code = subprocess.check_call(['snap'] + commands, env=environ)
except subprocess.CalledProcessError as e:
retry_count += + 1
if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
return_code = e.returncode
log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
sleep(SNAP_NO_LOCK_RETRY_DELAY)
return return_code
def snap_install(packages, *flags):
"""
Install a snap package.
:param packages: String or List String package name
:param flags: List String flags to pass to install command
:return: Integer return code from snap
"""
if type(packages) is not list:
packages = [packages]
flags = list(flags)
message = 'Installing snap(s) "%s"' % ', '.join(packages)
if flags:
message += ' with option(s) "%s"' % ', '.join(flags)
log(message, level='INFO')
return _snap_exec(['install'] + flags + packages)
def snap_remove(packages, *flags):
"""
Remove a snap package.
:param packages: String or List String package name
:param flags: List String flags to pass to remove command
:return: Integer return code from snap
"""
if type(packages) is not list:
packages = [packages]
flags = list(flags)
message = 'Removing snap(s) "%s"' % ', '.join(packages)
if flags:
message += ' with options "%s"' % ', '.join(flags)
log(message, level='INFO')
return _snap_exec(['remove'] + flags + packages)
def snap_refresh(packages, *flags):
"""
Refresh / Update snap package.
:param packages: String or List String package name
:param flags: List String flags to pass to refresh command
:return: Integer return code from snap
"""
if type(packages) is not list:
packages = [packages]
flags = list(flags)
message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
if flags:
message += ' with options "%s"' % ', '.join(flags)
log(message, level='INFO')
return _snap_exec(['refresh'] + flags + packages)

80
hooks/charmhelpers/fetch/ubuntu.py
View File

@ -116,8 +116,8 @@ CLOUD_ARCHIVE_POCKETS = {
}
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
def filter_installed_packages(packages):
@ -249,7 +249,8 @@ def add_source(source, key=None):
source.startswith('http') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
subprocess.check_call(['add-apt-repository', '--yes', source])
cmd = ['add-apt-repository', '--yes', source]
_run_with_retries(cmd)
elif source.startswith('cloud:'):
install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
@ -286,41 +287,60 @@ def add_source(source, key=None):
key])
def _run_apt_command(cmd, fatal=False):
"""Run an APT command.
Checks the output and retries if the fatal flag is set
to True.
def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
retry_message="", cmd_env=None):
"""Run a command and retry until success or max_retries is reached.
:param: cmd: str: The apt command to run.
:param: max_retries: int: The number of retries to attempt on a fatal
command. Defaults to CMD_RETRY_COUNT.
:param: retry_exitcodes: tuple: Optional additional exit codes to retry.
Defaults to retry on exit code 1.
:param: retry_message: str: Optional log prefix emitted during retries.
:param: cmd_env: dict: Environment variables to add to the command run.
"""
env = os.environ.copy()
if cmd_env:
env.update(cmd_env)
if not retry_message:
retry_message = "Failed executing '{}'".format(" ".join(cmd))
retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
retry_count = 0
result = None
retry_results = (None,) + retry_exitcodes
while result in retry_results:
try:
result = subprocess.check_call(cmd, env=env)
except subprocess.CalledProcessError as e:
retry_count = retry_count + 1
if retry_count > max_retries:
raise
result = e.returncode
log(retry_message)
time.sleep(CMD_RETRY_DELAY)
def _run_apt_command(cmd, fatal=False):
"""Run an apt command with optional retries.
:param: fatal: bool: Whether the command's output should be checked and
retried.
"""
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
# Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
cmd_env = {
'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
if fatal:
retry_count = 0
result = None
# If the command is considered "fatal", we need to retry if the apt
# lock was not acquired.
while result is None or result == APT_NO_LOCK:
try:
result = subprocess.check_call(cmd, env=env)
except subprocess.CalledProcessError as e:
retry_count = retry_count + 1
if retry_count > APT_NO_LOCK_RETRY_COUNT:
raise
result = e.returncode
log("Couldn't acquire DPKG lock. Will retry in {} seconds."
"".format(APT_NO_LOCK_RETRY_DELAY))
time.sleep(APT_NO_LOCK_RETRY_DELAY)
_run_with_retries(
cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
retry_message="Couldn't acquire DPKG lock")
else:
env = os.environ.copy()
env.update(cmd_env)
subprocess.call(cmd, env=env)

11
hooks/nova_compute_context.py
View File

@ -32,11 +32,9 @@ from charmhelpers.core.hookenv import (
relation_ids,
related_units,
service_name,
unit_get,
ERROR,
)
from charmhelpers.contrib.openstack.utils import (
get_host_ip,
get_os_version_package,
get_os_version_codename,
os_release,
@ -46,7 +44,7 @@ from charmhelpers.contrib.openstack.ip import (
resolve_address,
)
from charmhelpers.contrib.network.ip import (
get_ipv6_addr,
get_relation_ip,
)
# This is just a label and it must be consistent across
@ -599,10 +597,9 @@ class NeutronComputeContext(context.OSContextGenerator):
class HostIPContext(context.OSContextGenerator):
def __call__(self):
ctxt = {}
if config('prefer-ipv6'):
host_ip = get_ipv6_addr()[0]
else:
host_ip = get_host_ip(unit_get('private-address'))
# Use the address used in the cloud-compute relation in templates for
# this host
host_ip = get_relation_ip('cloud-compute')
if host_ip:
# NOTE: do not format this even for ipv6 (see bug 1499656)

14
hooks/nova_compute_hooks.py
View File

@ -33,7 +33,6 @@ from charmhelpers.core.hookenv import (
relation_get,
relation_set,
service_name,
unit_get,
UnregisteredHookError,
status_set,
)
@ -95,7 +94,7 @@ from nova_compute_utils import (
)
from charmhelpers.contrib.network.ip import (
get_ipv6_addr
get_relation_ip,
)
from charmhelpers.core.unitdata import kv
@ -244,7 +243,7 @@ def db_joined(rid=None):
relation_set(relation_id=rid,
nova_database=config('database'),
nova_username=config('database-user'),
nova_hostname=unit_get('private-address'))
nova_hostname=get_relation_ip('shared-db'))
@hooks.hook('pgsql-db-relation-joined')
@ -256,7 +255,8 @@ def pgsql_db_joined():
log(e, level=ERROR)
raise Exception(e)
relation_set(database=config('database'))
relation_set(**{'database': config('database'),
'private-address': get_relation_ip('psql-db')})
@hooks.hook('shared-db-relation-changed')
@ -302,10 +302,10 @@ def compute_joined(rid=None):
# record so won't get scanned based on private-address which is an IP
# add the hostname configured locally to the relation.
settings = {
'hostname': gethostname()
'hostname': gethostname(),
'private-address': get_relation_ip('cloud-compute'),
}
if config('prefer-ipv6'):
settings = {'private-address': get_ipv6_addr()[0]}
if migration_enabled():
auth_type = config('migration-auth-type')
settings['migration_auth_type'] = auth_type

59
tests/charmhelpers/contrib/amulet/utils.py
View File

@ -25,7 +25,6 @@ import uuid
import amulet
import distro_info
from distutils.version import LooseVersion
import six
from six.moves import configparser
if six.PY3:
@ -790,47 +789,26 @@ class AmuletUtils(object):
def run_action(self, unit_sentry, action,
_check_output=subprocess.check_output,
params=None):
"""Run the named action on a given unit sentry.
"""Translate to amulet's built in run_action(). Deprecated.
Run the named action on a given unit sentry.
params a dict of parameters to use
_check_output parameter is used for dependency injection.
_check_output parameter is no longer used
@return action_id.
"""
unit_id = unit_sentry.info["unit_name"]
# Note: There were significant CLI changes between 2.0 and 2.1
# This supports the 2.1 and 1.25.x only
# We are supporting the latest stable 1.x and 2.x
if self.juju_min_version("2.1"):
command = ["juju", "run-action", "--format=json", unit_id, action]
else:
command = ["juju", "action", "do", "--format=json",
unit_id, action]
if params is not None:
for key, value in params.iteritems():
command.append("{}={}".format(key, value))
self.log.info("Running command: %s\n" % " ".join(command))
output = _check_output(command, universal_newlines=True)
data = json.loads(output)
action_id = data[u'Action queued with id']
return action_id
self.log.warn('charmhelpers.contrib.amulet.utils.run_action has been '
'deprecated for amulet.run_action')
return unit_sentry.run_action(action, action_args=params)
def wait_on_action(self, action_id, _check_output=subprocess.check_output):
"""Wait for a given action, returning if it completed or not.
_check_output parameter is used for dependency injection.
action_id a string action uuid
_check_output parameter is no longer used
"""
# Note: There were significant CLI changes between 2.0 and 2.1
# This supports the 2.1 and 1.25.x only
# We are supporting the latest stable 1.x and 2.x
if self.juju_min_version("2.1"):
command = ["juju", "show-action-output", "--format=json",
"--wait=0", action_id]
else:
command = ["juju", "action", "fetch", "--format=json", "--wait=0",
action_id]
output = _check_output(command, universal_newlines=True)
data = json.loads(output)
data = amulet.actions.get_action_output(action_id, full_output=True)
return data.get(u"status") == "completed"
def status_get(self, unit):
@ -841,20 +819,3 @@ class AmuletUtils(object):
return ("unknown", "")
status = json.loads(raw_status)
return (status["status"], status["message"])
def juju_version(self):
"""Return Juju version
@reutrns string version
"""
cmd = ['juju', 'version']
return subprocess.check_output(cmd)
def juju_min_version(self, minimum_version='2.1'):
"""Return True if the Juju version is at least the provided version
@param minimum_version: string semantic version
@returns boolean
"""
return (LooseVersion(self.juju_version()) >=
LooseVersion(minimum_version))

1
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -435,7 +435,6 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
self.log.debug("~~~~~~~~~~~~~~VERSION: {} ~~~~~~~~~~~~~~~~~~~~~".format(novaclient.__version__[0]))
if novaclient.__version__[0] >= "7":
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, password=password,

13
unit_tests/test_nova_compute_contexts.py
View File

@ -29,9 +29,9 @@ TO_PATCH = [
'config',
'log',
'_save_flag_file',
'unit_get',
'lsb_release',
'os_release',
'get_relation_ip',
]
NEUTRON_CONTEXT = {
@ -358,20 +358,19 @@ class NovaComputeContextTests(CharmTestCase):
@patch('subprocess.call')
def test_host_IP_context(self, _call):
self.log = fake_log
self.unit_get.return_value = '172.24.0.79'
self.get_relation_ip.return_value = '172.24.0.79'
host_ip = context.HostIPContext()
self.assertEqual({'host_ip': '172.24.0.79'}, host_ip())
self.unit_get.assert_called_with('private-address')
self.get_relation_ip.assert_called_with('cloud-compute')
@patch.object(context, 'get_ipv6_addr')
@patch('subprocess.call')
def test_host_IP_context_ipv6(self, _call, mock_get_ipv6_addr):
def test_host_IP_context_ipv6(self, _call):
self.log = fake_log
self.test_config.set('prefer-ipv6', True)
mock_get_ipv6_addr.return_value = ['2001:db8:0:1::2']
self.get_relation_ip.return_value = '2001:db8:0:1::2'
host_ip = context.HostIPContext()
self.assertEqual({'host_ip': '2001:db8:0:1::2'}, host_ip())
self.assertTrue(mock_get_ipv6_addr.called)
self.assertTrue(self.get_relation_ip.called)
def test_metadata_service_ctxt(self):
self.relation_ids.return_value = 'neutron-plugin:0'

61
unit_tests/test_nova_compute_hooks.py
View File

@ -50,7 +50,6 @@ TO_PATCH = [
'relation_set',
'service_name',
'related_units',
'unit_get',
# charmhelpers.core.host
'apt_install',
'apt_purge',
@ -61,6 +60,8 @@ TO_PATCH = [
# charmhelpers.contrib.openstack.utils
'configure_installation_source',
'openstack_upgrade_available',
# charmhelpers.contrib.network.ip
'get_relation_ip',
# nova_compute_context
'nova_metadata_requirement',
# nova_compute_utils
@ -108,6 +109,7 @@ class NovaComputeRelationsTests(CharmTestCase):
self.filter_installed_packages.side_effect = \
MagicMock(side_effect=lambda pkgs: pkgs)
self.gethostname.return_value = 'testserver'
self.get_relation_ip.return_value = '10.0.0.50'
def test_install_hook(self):
repo = 'cloud:precise-grizzly'
@ -344,20 +346,19 @@ class NovaComputeRelationsTests(CharmTestCase):
configs.write.call_args_list)
def test_db_joined(self):
self.unit_get.return_value = 'nova.foohost.com'
self.is_relation_made.return_value = False
hooks.db_joined()
self.relation_set.assert_called_with(relation_id=None,
nova_database='nova',
nova_username='nova',
nova_hostname='nova.foohost.com')
self.unit_get.assert_called_with('private-address')
nova_hostname='10.0.0.50')
self.get_relation_ip.assert_called_with('shared-db')
def test_postgresql_db_joined(self):
self.unit_get.return_value = 'nova.foohost.com'
self.is_relation_made.return_value = False
hooks.pgsql_db_joined()
self.relation_set.assert_called_with(database='nova'),
self.relation_set.assert_called_with(**{
'database': 'nova', 'private-address': '10.0.0.50'})
def test_db_joined_with_postgresql(self):
self.is_relation_made.return_value = True
@ -446,36 +447,40 @@ class NovaComputeRelationsTests(CharmTestCase):
self.test_config.set('migration-auth-type', 'ssh')
self.public_ssh_key.return_value = 'foo'
hooks.compute_joined()
self.relation_set.assert_called_with(
relation_id=None,
ssh_public_key='foo',
migration_auth_type='ssh',
hostname='testserver',
)
self.relation_set.assert_called_with(**{
'relation_id': None,
'ssh_public_key': 'foo',
'migration_auth_type': 'ssh',
'hostname': 'testserver',
'private-address': '10.0.0.50',
})
hooks.compute_joined(rid='cloud-compute:2')
self.relation_set.assert_called_with(
relation_id='cloud-compute:2',
ssh_public_key='foo',
migration_auth_type='ssh',
hostname='testserver',
)
self.relation_set.assert_called_with(**{
'relation_id': 'cloud-compute:2',
'ssh_public_key': 'foo',
'migration_auth_type': 'ssh',
'hostname': 'testserver',
'private-address': '10.0.0.50',
})
def test_compute_joined_with_resize(self):
self.migration_enabled.return_value = False
self.test_config.set('enable-resize', True)
self.public_ssh_key.return_value = 'bar'
hooks.compute_joined()
self.relation_set.assert_called_with(
relation_id=None,
nova_ssh_public_key='bar',
hostname='testserver',
)
self.relation_set.assert_called_with(**{
'relation_id': None,
'nova_ssh_public_key': 'bar',
'hostname': 'testserver',
'private-address': '10.0.0.50',
})
hooks.compute_joined(rid='cloud-compute:2')
self.relation_set.assert_called_with(
relation_id='cloud-compute:2',
nova_ssh_public_key='bar',
hostname='testserver',
)
self.relation_set.assert_called_with(**{
'relation_id': 'cloud-compute:2',
'nova_ssh_public_key': 'bar',
'hostname': 'testserver',
'private-address': '10.0.0.50',
})
def test_compute_changed(self):
hooks.compute_changed()