diff --git a/Makefile b/Makefile index 5b02fbfa..db4e2c25 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,7 @@ PYTHON := /usr/bin/env python lint: - @flake8 --exclude hooks/charmhelpers hooks - @flake8 --exclude hooks/charmhelpers unit_tests + @flake8 --exclude hooks/charmhelpers hooks unit_tests tests @charm proof unit_test: @@ -17,6 +16,16 @@ bin/charm_helpers_sync.py: sync: bin/charm_helpers_sync.py @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml + +test: + @echo Starting Amulet tests... + # coreycb note: The -v should only be temporary until Amulet sends + # raise_status() messages to stderr: + # https://bugs.launchpad.net/amulet/+bug/1320357 + @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ + 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \ + 16-basic-trusty-juno publish: lint unit_test bzr push lp:charms/neutron-openvswitch diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml new file mode 100644 index 00000000..48b12f6f --- /dev/null +++ b/charm-helpers-tests.yaml @@ -0,0 +1,5 @@ +branch: lp:charm-helpers +destination: tests/charmhelpers +include: + - contrib.amulet + - contrib.openstack.amulet diff --git a/config.yaml b/config.yaml index c424956c..94f19c67 100644 --- a/config.yaml +++ b/config.yaml @@ -25,8 +25,10 @@ options: type: string default: description: | - The data port will be added to br-data and will allow usage of flat or VLAN - network types + Space-delimited list of bridge:port mappings. Ports will be added to + their corresponding bridge. The bridges will allow usage of flat or + VLAN network types with Neutron and should match this defined in + bridge-mappings. disable-security-groups: type: boolean default: false @@ -36,6 +38,17 @@ options: . BE CAREFUL - this option allows you to disable all port level security within an OpenStack cloud. + bridge-mappings: + type: string + default: 'physnet1:br-data' + description: | + Space-delimited list of ML2 data bridge mappings with format + :. + vlan-ranges: + type: string + default: "physnet1:1000:2000" + description: | + Space-delimited list of network provider vlan id ranges. # Network configuration options # by default all access is over 'private-address' os-data-network: diff --git a/hooks/neutron_ovs_context.py b/hooks/neutron_ovs_context.py index 513c577b..a7192a4e 100644 --- a/hooks/neutron_ovs_context.py +++ b/hooks/neutron_ovs_context.py @@ -4,24 +4,26 @@ from charmhelpers.core.hookenv import ( config, unit_get, ) -from charmhelpers.contrib.network.ip import ( - get_address_in_network, -) from charmhelpers.contrib.openstack.ip import resolve_address -from charmhelpers.core.host import list_nics, get_nic_hwaddr from charmhelpers.contrib.openstack import context -from charmhelpers.core.host import service_running, service_start +from charmhelpers.core.host import ( + service_running, + service_start, + service_restart, +) from charmhelpers.contrib.network.ovs import add_bridge, add_bridge_port from charmhelpers.contrib.openstack.utils import get_host_ip +from charmhelpers.contrib.network.ip import get_address_in_network from charmhelpers.contrib.openstack.context import ( OSContextGenerator, NeutronAPIContext, + DataPortContext, +) +from charmhelpers.contrib.openstack.neutron import ( + parse_bridge_mappings, + parse_vlan_range_mappings, ) - -import re - OVS_BRIDGE = 'br-int' -DATA_BRIDGE = 'br-data' class OVSPluginContext(context.NeutronContext): @@ -37,34 +39,28 @@ class OVSPluginContext(context.NeutronContext): @property def neutron_security_groups(self): + if config('disable-security-groups'): + return False neutron_api_settings = NeutronAPIContext()() return neutron_api_settings['neutron_security_groups'] - def get_data_port(self): - data_ports = config('data-port') - if not data_ports: - return None - hwaddrs = {} - for nic in list_nics(['eth', 'bond']): - hwaddrs[get_nic_hwaddr(nic).lower()] = nic - mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) - for entry in data_ports.split(): - entry = entry.strip().lower() - if re.match(mac_regex, entry): - if entry in hwaddrs: - return hwaddrs[entry] - else: - return entry - return None - def _ensure_bridge(self): if not service_running('openvswitch-switch'): service_start('openvswitch-switch') + add_bridge(OVS_BRIDGE) - add_bridge(DATA_BRIDGE) - data_port = self.get_data_port() - if data_port: - add_bridge_port(DATA_BRIDGE, data_port, promisc=True) + + portmaps = DataPortContext()() + bridgemaps = parse_bridge_mappings(config('bridge-mappings')) + for provider, br in bridgemaps.iteritems(): + add_bridge(br) + + if not portmaps or br not in portmaps: + continue + + add_bridge_port(br, portmaps[br], promisc=True) + + service_restart('os-charm-phy-nic-mtu') def ovs_ctxt(self): # In addition to generating config context, ensure the OVS service @@ -91,6 +87,25 @@ class OVSPluginContext(context.NeutronContext): ovs_ctxt['use_syslog'] = conf['use-syslog'] ovs_ctxt['verbose'] = conf['verbose'] ovs_ctxt['debug'] = conf['debug'] + + net_dev_mtu = neutron_api_settings.get('network_device_mtu') + if net_dev_mtu: + # neutron.conf + ovs_ctxt['network_device_mtu'] = net_dev_mtu + # ml2 conf + ovs_ctxt['veth_mtu'] = net_dev_mtu + + mappings = config('bridge-mappings') + if mappings: + ovs_ctxt['bridge_mappings'] = mappings + + vlan_ranges = config('vlan-ranges') + vlan_range_mappings = parse_vlan_range_mappings(config('vlan-ranges')) + if vlan_ranges: + providers = vlan_range_mappings.keys() + ovs_ctxt['network_providers'] = ' '.join(providers) + ovs_ctxt['vlan_ranges'] = vlan_ranges + return ovs_ctxt diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 7ad866a3..ee7435bd 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -22,6 +22,8 @@ ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR EXT_PORT_CONF = '/etc/init/ext-port.conf' NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini" DVR_PACKAGES = ['neutron-vpn-agent'] +PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf' +TEMPLATES = 'templates/' BASE_RESOURCE_MAP = OrderedDict([ (NEUTRON_CONF, { @@ -33,6 +35,10 @@ BASE_RESOURCE_MAP = OrderedDict([ 'services': ['neutron-plugin-openvswitch-agent'], 'contexts': [neutron_ovs_context.OVSPluginContext()], }), + (PHY_NIC_MTU_CONF, { + 'services': ['os-charm-phy-nic-mtu'], + 'contexts': [context.PhyNICMTUContext()], + }), ]) DVR_RESOURCE_MAP = OrderedDict([ (NEUTRON_L3_AGENT_CONF, { diff --git a/templates/icehouse/ml2_conf.ini b/templates/icehouse/ml2_conf.ini index 56be2052..f2aa23e3 100644 --- a/templates/icehouse/ml2_conf.ini +++ b/templates/icehouse/ml2_conf.ini @@ -16,19 +16,22 @@ tunnel_id_ranges = 1:1000 vni_ranges = 1001:2000 [ml2_type_vlan] -network_vlan_ranges = physnet1:1000:2000 +network_vlan_ranges = {{ vlan_ranges }} [ml2_type_flat] -flat_networks = physnet1 +flat_networks = {{ network_providers }} [ovs] enable_tunneling = True local_ip = {{ local_ip }} -bridge_mappings = physnet1:br-data +bridge_mappings = {{ bridge_mappings }} [agent] tunnel_types = {{ overlay_network_type }} l2_population = {{ l2_population }} +{% if veth_mtu -%} +veth_mtu = {{ veth_mtu }} +{% endif %} [securitygroup] {% if neutron_security_groups -%} diff --git a/templates/icehouse/neutron.conf b/templates/icehouse/neutron.conf index 964f6dce..b0e35f9a 100644 --- a/templates/icehouse/neutron.conf +++ b/templates/icehouse/neutron.conf @@ -1,4 +1,4 @@ -# grizzly +# icehouse ############################################################################### # [ WARNING ] # Configuration file maintained by Juju. Local changes may be overwritten. @@ -12,7 +12,9 @@ state_path = /var/lib/neutron lock_path = $state_path/lock bind_host = 0.0.0.0 bind_port = 9696 - +{% if network_device_mtu -%} +network_device_mtu = {{ network_device_mtu }} +{% endif -%} {% if core_plugin -%} core_plugin = {{ core_plugin }} {% endif -%} diff --git a/templates/os-charm-phy-nic-mtu.conf b/templates/os-charm-phy-nic-mtu.conf new file mode 100644 index 00000000..06d1967b --- /dev/null +++ b/templates/os-charm-phy-nic-mtu.conf @@ -0,0 +1,22 @@ +description "Enabling Quantum external networking port" + +start on runlevel [2345] + +task + +script + devs="{{ devs }}" + mtu="{{ mtu }}" + tmpfile=`mktemp` + echo $devs > $tmpfile + if [ -n "$mtu" ]; then + while read -r dev; do + [ -n "$dev" ] || continue + rc=0 + # Try all devices before exiting with error + ip link set $dev mtu $mtu || rc=$? + done < $tmpfile + rm $tmpfile + [ $rc = 0 ] || exit $rc + fi +end script \ No newline at end of file diff --git a/tests/00-setup b/tests/00-setup new file mode 100755 index 00000000..06cfdb07 --- /dev/null +++ b/tests/00-setup @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +sudo add-apt-repository --yes ppa:juju/stable +sudo apt-get update --yes +sudo apt-get install --yes python-amulet \ + python-neutronclient \ + python-keystoneclient \ + python-novaclient \ + python-glanceclient diff --git a/tests/14-basic-precise-icehouse b/tests/14-basic-precise-icehouse new file mode 100755 index 00000000..8b9ce35a --- /dev/null +++ b/tests/14-basic-precise-icehouse @@ -0,0 +1,11 @@ +#!/usr/bin/python +# NeutronOVSBasicDeployment +"""Amulet tests on a basic neutron-openvswitch deployment on precise-icehouse.""" + +from basic_deployment import NeutronOVSBasicDeployment + +if __name__ == '__main__': + deployment = NeutronOVSBasicDeployment(series='precise', + openstack='cloud:precise-icehouse', + source='cloud:precise-updates/icehouse') + deployment.run_tests() diff --git a/tests/15-basic-trusty-icehouse b/tests/15-basic-trusty-icehouse new file mode 100755 index 00000000..67f2191c --- /dev/null +++ b/tests/15-basic-trusty-icehouse @@ -0,0 +1,9 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-openvswitch deployment on trusty-icehouse.""" + +from basic_deployment import NeutronOVSBasicDeployment + +if __name__ == '__main__': + deployment = NeutronOVSBasicDeployment(series='trusty') + deployment.run_tests() diff --git a/tests/16-basic-trusty-juno b/tests/16-basic-trusty-juno new file mode 100755 index 00000000..dd6ba7b6 --- /dev/null +++ b/tests/16-basic-trusty-juno @@ -0,0 +1,11 @@ +#!/usr/bin/python + +"""Amulet tests on a basic neutron-openvswitch deployment on trusty-juno.""" + +from basic_deployment import NeutronOVSBasicDeployment + +if __name__ == '__main__': + deployment = NeutronOVSBasicDeployment(series='trusty', + openstack='cloud:trusty-juno', + source='cloud:trusty-updates/juno') + deployment.run_tests() diff --git a/tests/README b/tests/README new file mode 100644 index 00000000..9c3bdbcf --- /dev/null +++ b/tests/README @@ -0,0 +1,53 @@ +This directory provides Amulet tests that focus on verification of +neutron-openvswitch deployments. + +In order to run tests, you'll need charm-tools installed (in addition to +juju, of course): + sudo add-apt-repository ppa:juju/stable + sudo apt-get update + sudo apt-get install charm-tools + +If you use a web proxy server to access the web, you'll need to set the +AMULET_HTTP_PROXY environment variable to the http URL of the proxy server. + +The following examples demonstrate different ways that tests can be executed. +All examples are run from the charm's root directory. + + * To run all tests (starting with 00-setup): + + make test + + * To run a specific test module (or modules): + + juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse + + * To run a specific test module (or modules), and keep the environment + deployed after a failure: + + juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse + + * To re-run a test module against an already deployed environment (one + that was deployed by a previous call to 'juju test --set-e'): + + ./tests/15-basic-trusty-icehouse + +For debugging and test development purposes, all code should be idempotent. +In other words, the code should have the ability to be re-run without changing +the results beyond the initial run. This enables editing and re-running of a +test module against an already deployed environment, as described above. + +Manual debugging tips: + + * Set the following env vars before using the OpenStack CLI as admin: + export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 + export OS_TENANT_NAME=admin + export OS_USERNAME=admin + export OS_PASSWORD=openstack + export OS_REGION_NAME=RegionOne + + * Set the following env vars before using the OpenStack CLI as demoUser: + export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0 + export OS_TENANT_NAME=demoTenant + export OS_USERNAME=demoUser + export OS_PASSWORD=password + export OS_REGION_NAME=RegionOne diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py new file mode 100644 index 00000000..6fee544d --- /dev/null +++ b/tests/basic_deployment.py @@ -0,0 +1,201 @@ +#!/usr/bin/python + +import amulet +import time + +from charmhelpers.contrib.openstack.amulet.deployment import ( + OpenStackAmuletDeployment +) + +from charmhelpers.contrib.openstack.amulet.utils import ( + OpenStackAmuletUtils, + DEBUG, # flake8: noqa + ERROR +) + +# Use DEBUG to turn on debug logging +u = OpenStackAmuletUtils(ERROR) + +# XXX Tests inspecting relation data from the perspective of the +# neutron-openvswitch are missing because amulet sentries aren't created for +# subordinates Bug#1421388 + + +class NeutronOVSBasicDeployment(OpenStackAmuletDeployment): + """Amulet tests on a basic neutron-openvswtich deployment.""" + + def __init__(self, series, openstack=None, source=None, stable=False): + """Deploy the entire test environment.""" + super(NeutronOVSBasicDeployment, self).__init__(series, openstack, + source, stable) + self._add_services() + self._add_relations() + self._configure_services() + self._deploy() + self._initialize_tests() + + def _add_services(self): + """Add services + + Add the services that we're testing, where neutron-openvswitch is local, + and the rest of the service are from lp branches that are + compatible with the local charm (e.g. stable or next). + """ + this_service = {'name': 'neutron-openvswitch'} + other_services = [{'name': 'nova-compute'}, + {'name': 'rabbitmq-server'}, + {'name': 'neutron-api'}] + super(NeutronOVSBasicDeployment, self)._add_services(this_service, + other_services) + + def _add_relations(self): + """Add all of the relations for the services.""" + relations = { + 'neutron-openvswitch:amqp': 'rabbitmq-server:amqp', + 'neutron-openvswitch:neutron-plugin': + 'nova-compute:neutron-plugin', + 'neutron-openvswitch:neutron-plugin-api': + 'neutron-api:neutron-plugin-api', + } + super(NeutronOVSBasicDeployment, self)._add_relations(relations) + + def _configure_services(self): + """Configure all of the services.""" + configs = {} + super(NeutronOVSBasicDeployment, self)._configure_services(configs) + + def _initialize_tests(self): + """Perform final initialization before tests get run.""" + # Access the sentries for inspecting service units + self.compute_sentry = self.d.sentry.unit['nova-compute/0'] + self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0'] + self.neutron_api_sentry = self.d.sentry.unit['neutron-api/0'] + + def test_services(self): + """Verify the expected services are running on the corresponding + service units.""" + + commands = { + self.compute_sentry: ['status nova-compute'], + self.rabbitmq_sentry: ['service rabbitmq-server status'], + self.neutron_api_sentry: ['status neutron-server'], + } + + ret = u.validate_services(commands) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_rabbitmq_amqp_relation(self): + """Verify data in rabbitmq-server/neutron-openvswitch amqp relation""" + unit = self.rabbitmq_sentry + relation = ['amqp', 'neutron-openvswitch:amqp'] + expected = { + 'private-address': u.valid_ip, + 'password': u.not_null, + 'hostname': u.valid_ip + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('rabbitmq amqp', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_nova_compute_relation(self): + """Verify the nova-compute to neutron-openvswitch relation data""" + unit = self.compute_sentry + relation = ['neutron-plugin', 'neutron-openvswitch:neutron-plugin'] + expected = { + 'private-address': u.valid_ip, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('nova-compute neutron-plugin', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_neutron_api_relation(self): + """Verify the neutron-api to neutron-openvswitch relation data""" + unit = self.neutron_api_sentry + relation = ['neutron-plugin-api', + 'neutron-openvswitch:neutron-plugin-api'] + expected = { + 'private-address': u.valid_ip, + } + + ret = u.validate_relation_data(unit, relation, expected) + if ret: + message = u.relation_error('neutron-api neutron-plugin-api', ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def process_ret(self, ret=None, message=None): + if ret: + amulet.raise_status(amulet.FAIL, msg=message) + + def check_ml2_setting_propagation(self, service, charm_key, + config_file_key, vpair, + section): + unit = self.compute_sentry + conf = "/etc/neutron/plugins/ml2/ml2_conf.ini" + for value in vpair: + self.d.configure(service, {charm_key: value}) + time.sleep(30) + ret = u.validate_config_data(unit, conf, section, + {config_file_key: value}) + msg = "Propagation error, expected %s=%s" % (config_file_key, + value) + self.process_ret(ret=ret, message=msg) + + def test_l2pop_propagation(self): + """Verify that neutron-api l2pop setting propagates to neutron-ovs""" + self.check_ml2_setting_propagation('neutron-api', + 'l2-population', + 'l2_population', + ['False', 'True'], + 'agent') + + def test_nettype_propagation(self): + """Verify that neutron-api nettype setting propagates to neutron-ovs""" + self.check_ml2_setting_propagation('neutron-api', + 'overlay-network-type', + 'tunnel_types', + ['vxlan', 'gre'], + 'agent') + + def test_secgroup_propagation_local_override(self): + """Verify disable-security-groups overrides what neutron-api says""" + unit = self.compute_sentry + conf = "/etc/neutron/plugins/ml2/ml2_conf.ini" + self.d.configure('neutron-api', {'neutron-security-groups': 'True'}) + self.d.configure('neutron-openvswitch', + {'disable-security-groups': 'True'}) + time.sleep(30) + ret = u.validate_config_data(unit, conf, 'securitygroup', + {'enable_security_group': 'False'}) + msg = "Propagation error, expected %s=%s" % ('enable_security_group', + 'False') + self.process_ret(ret=ret, message=msg) + self.d.configure('neutron-openvswitch', + {'disable-security-groups': 'False'}) + self.d.configure('neutron-api', {'neutron-security-groups': 'True'}) + time.sleep(30) + ret = u.validate_config_data(unit, conf, 'securitygroup', + {'enable_security_group': 'True'}) + + def test_z_restart_on_config_change(self): + """Verify that the specified services are restarted when the config + is changed. + + Note(coreycb): The method name with the _z_ is a little odd + but it forces the test to run last. It just makes things + easier because restarting services requires re-authorization. + """ + conf = '/etc/neutron/neutron.conf' + self.d.configure('neutron-openvswitch', {'use-syslog': 'True'}) + if not u.service_restarted(self.compute_sentry, + 'neutron-openvswitch-agent', conf, + pgrep_full=True, sleep_time=60): + self.d.configure('neutron-openvswitch', {'use-syslog': 'False'}) + msg = ('service neutron-openvswitch-agent did not restart after ' + 'config change') + amulet.raise_status(amulet.FAIL, msg=msg) + self.d.configure('neutron-openvswitch', {'use-syslog': 'False'}) diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py new file mode 100644 index 00000000..f72e7f84 --- /dev/null +++ b/tests/charmhelpers/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# Bootstrap charm-helpers, installing its dependencies if necessary using +# only standard libraries. +import subprocess +import sys + +try: + import six # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) + import six # flake8: noqa + +try: + import yaml # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + import yaml # flake8: noqa diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py new file mode 100644 index 00000000..d1400a02 --- /dev/null +++ b/tests/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py new file mode 100644 index 00000000..d1400a02 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py new file mode 100644 index 00000000..367d6b47 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -0,0 +1,93 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import amulet +import os +import six + + +class AmuletDeployment(object): + """Amulet deployment. + + This class provides generic Amulet deployment and test runner + methods. + """ + + def __init__(self, series=None): + """Initialize the deployment environment.""" + self.series = None + + if series: + self.series = series + self.d = amulet.Deployment(series=self.series) + else: + self.d = amulet.Deployment() + + def _add_services(self, this_service, other_services): + """Add services. + + Add services to the deployment where this_service is the local charm + that we're testing and other_services are the other services that + are being used in the local amulet tests. + """ + if this_service['name'] != os.path.basename(os.getcwd()): + s = this_service['name'] + msg = "The charm's root directory name needs to be {}".format(s) + amulet.raise_status(amulet.FAIL, msg=msg) + + if 'units' not in this_service: + this_service['units'] = 1 + + self.d.add(this_service['name'], units=this_service['units']) + + for svc in other_services: + if 'location' in svc: + branch_location = svc['location'] + elif self.series: + branch_location = 'cs:{}/{}'.format(self.series, svc['name']), + else: + branch_location = None + + if 'units' not in svc: + svc['units'] = 1 + + self.d.add(svc['name'], charm=branch_location, units=svc['units']) + + def _add_relations(self, relations): + """Add all of the relations for the services.""" + for k, v in six.iteritems(relations): + self.d.relate(k, v) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in six.iteritems(configs): + self.d.configure(service, config) + + def _deploy(self): + """Deploy environment and wait for all hooks to finish executing.""" + try: + self.d.setup(timeout=900) + self.d.sentry.wait(timeout=900) + except amulet.helpers.TimeoutError: + amulet.raise_status(amulet.FAIL, msg="Deployment timed out") + except Exception: + raise + + def run_tests(self): + """Run all of the methods that are prefixed with 'test_'.""" + for test in dir(self): + if test.startswith('test_'): + getattr(self, test)() diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py new file mode 100644 index 00000000..65219d33 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -0,0 +1,314 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import ConfigParser +import io +import logging +import re +import sys +import time + +import six + + +class AmuletUtils(object): + """Amulet utilities. + + This class provides common utility functions that are used by Amulet + tests. + """ + + def __init__(self, log_level=logging.ERROR): + self.log = self.get_logger(level=log_level) + + def get_logger(self, name="amulet-logger", level=logging.DEBUG): + """Get a logger object that will log to stdout.""" + log = logging + logger = log.getLogger(name) + fmt = log.Formatter("%(asctime)s %(funcName)s " + "%(levelname)s: %(message)s") + + handler = log.StreamHandler(stream=sys.stdout) + handler.setLevel(level) + handler.setFormatter(fmt) + + logger.addHandler(handler) + logger.setLevel(level) + + return logger + + def valid_ip(self, ip): + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip): + return True + else: + return False + + def valid_url(self, url): + p = re.compile( + r'^(?:http|ftp)s?://' + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa + r'localhost|' + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', + re.IGNORECASE) + if p.match(url): + return True + else: + return False + + def validate_services(self, commands): + """Validate services. + + Verify the specified services are running on the corresponding + service units. + """ + for k, v in six.iteritems(commands): + for cmd in v: + output, code = k.run(cmd) + if code != 0: + return "command `{}` returned {}".format(cmd, str(code)) + return None + + def _get_config(self, unit, filename): + """Get a ConfigParser object for parsing a unit's config file.""" + file_contents = unit.file_contents(filename) + config = ConfigParser.ConfigParser() + config.readfp(io.StringIO(file_contents)) + return config + + def validate_config_data(self, sentry_unit, config_file, section, + expected): + """Validate config file data. + + Verify that the specified section of the config file contains + the expected option key:value pairs. + """ + config = self._get_config(sentry_unit, config_file) + + if section != 'DEFAULT' and not config.has_section(section): + return "section [{}] does not exist".format(section) + + for k in expected.keys(): + if not config.has_option(section, k): + return "section [{}] is missing option {}".format(section, k) + if config.get(section, k) != expected[k]: + return "section [{}] {}:{} != expected {}:{}".format( + section, k, config.get(section, k), k, expected[k]) + return None + + def _validate_dict_data(self, expected, actual): + """Validate dictionary data. + + Compare expected dictionary data vs actual dictionary data. + The values in the 'expected' dictionary can be strings, bools, ints, + longs, or can be a function that evaluate a variable and returns a + bool. + """ + for k, v in six.iteritems(expected): + if k in actual: + if (isinstance(v, six.string_types) or + isinstance(v, bool) or + isinstance(v, six.integer_types)): + if v != actual[k]: + return "{}:{}".format(k, actual[k]) + elif not v(actual[k]): + return "{}:{}".format(k, actual[k]) + else: + return "key '{}' does not exist".format(k) + return None + + def validate_relation_data(self, sentry_unit, relation, expected): + """Validate actual relation data based on expected relation data.""" + actual = sentry_unit.relation(relation[0], relation[1]) + self.log.debug('actual: {}'.format(repr(actual))) + return self._validate_dict_data(expected, actual) + + def _validate_list_data(self, expected, actual): + """Compare expected list vs actual list data.""" + for e in expected: + if e not in actual: + return "expected item {} not found in actual list".format(e) + return None + + def not_null(self, string): + if string is not None: + return True + else: + return False + + def _get_file_mtime(self, sentry_unit, filename): + """Get last modification time of file.""" + return sentry_unit.file_stat(filename)['mtime'] + + def _get_dir_mtime(self, sentry_unit, directory): + """Get last modification time of directory.""" + return sentry_unit.directory_stat(directory)['mtime'] + + def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): + """Get process' start time. + + Determine start time of the process based on the last modification + time of the /proc/pid directory. If pgrep_full is True, the process + name is matched against the full command line. + """ + if pgrep_full: + cmd = 'pgrep -o -f {}'.format(service) + else: + cmd = 'pgrep -o {}'.format(service) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) + + def service_restarted(self, sentry_unit, service, filename, + pgrep_full=False, sleep_time=20): + """Check if service was restarted. + + Compare a service's start time vs a file's last modification time + (such as a config file for that service) to determine if the service + has been restarted. + """ + time.sleep(sleep_time) + if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= + self._get_file_mtime(sentry_unit, filename)): + return True + else: + return False + + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + + def relation_error(self, name, data): + return 'unexpected relation data in {} - {}'.format(name, data) + + def endpoint_error(self, name, data): + return 'unexpected endpoint data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py new file mode 100644 index 00000000..d1400a02 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 00000000..d1400a02 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 00000000..0cfeaa4c --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,111 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """OpenStack amulet deployment. + + This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms. + """ + + def __init__(self, series=None, openstack=None, source=None, stable=True): + """Initialize the deployment environment.""" + super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source + self.stable = stable + # Note(coreycb): this needs to be changed when new next branches come + # out. + self.current_next = "trusty" + + def _determine_branch_locations(self, other_services): + """Determine the branch locations for the other services. + + Determine if the local branch being tested is derived from its + stable or next (dev) branch, and based on this, use the corresonding + stable or next branches for the other_services.""" + base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] + + if self.stable: + for svc in other_services: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + for svc in other_services: + if svc['name'] in base_charms: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + temp = 'lp:~openstack-charmers/charms/{}/{}/next' + svc['location'] = temp.format(self.current_next, + svc['name']) + return other_services + + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin/source.""" + other_services = self._determine_branch_locations(other_services) + + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', + 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] + + if self.openstack: + for svc in services: + if svc['name'] not in use_source + ignore: + config = {'openstack-origin': self.openstack} + self.d.configure(svc['name'], config) + + if self.source: + for svc in services: + if svc['name'] in use_source and svc['name'] not in ignore: + config = {'source': self.source} + self.d.configure(svc['name'], config) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in six.iteritems(configs): + self.d.configure(service, config) + + def _get_openstack_release(self): + """Get openstack release. + + Return an integer representing the enum value of the openstack + release. + """ + (self.precise_essex, self.precise_folsom, self.precise_grizzly, + self.precise_havana, self.precise_icehouse, + self.trusty_icehouse) = range(6) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse} + return releases[(self.series, self.openstack)] diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 00000000..9c3d918a --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,294 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +import six + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """OpenStack amulet utilities. + + This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms. + """ + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate endpoint data. + + Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint. + """ + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if (admin_port in ep.adminurl and + internal_port in ep.internalurl and + public_port in ep.publicurl): + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate service catalog endpoint data. + + Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in six.iteritems(expected): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate tenant data. + + Validate a list of actual tenant data vs list of expected tenant + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e['name']) + return ret + + def validate_role_data(self, expected, actual): + """Validate role data. + + Validate a list of actual role data vs a list of expected role + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e['name']) + return ret + + def validate_user_data(self, expected, actual): + """Validate user data. + + Validate a list of actual user data vs a list of expected user + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e['name']) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate flavor data. + + Validate a list of actual flavors vs a list of expected flavors. + """ + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists.""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + unit = keystone_sentry + service_ip = unit.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "cirros-{}-x86_64-disk.img".format(version) + local_path = os.path.join('tests', cirros_img) + + if not os.path.exists(local_path): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, local_path) + f.close() + + with open(local_path) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + count = 1 + status = image.status + while status != 'active' and count < 10: + time.sleep(3) + image = glance.images.get(image.id) + status = image.status + self.log.debug('image status: {}'.format(status)) + count += 1 + + if status != 'active': + self.log.error('image creation timed out') + return None + + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + num_before = len(list(glance.images.list())) + glance.images.delete(image) + + count = 1 + num_after = len(list(glance.images.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(glance.images.list())) + self.log.debug('number of images: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('image deletion timed out') + return False + + return True + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status != 'ACTIVE' and count < 60: + time.sleep(3) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status != 'ACTIVE': + self.log.error('instance creation timed out') + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + num_before = len(list(nova.servers.list())) + nova.servers.delete(instance) + + count = 1 + num_after = len(list(nova.servers.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(nova.servers.list())) + self.log.debug('number of instances: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('instance deletion timed out') + return False + + return True diff --git a/unit_tests/test_neutron_ovs_context.py b/unit_tests/test_neutron_ovs_context.py index 5de00d0f..c0602396 100644 --- a/unit_tests/test_neutron_ovs_context.py +++ b/unit_tests/test_neutron_ovs_context.py @@ -5,9 +5,6 @@ from mock import patch import neutron_ovs_context as context import charmhelpers TO_PATCH = [ - 'relation_get', - 'relation_ids', - 'related_units', 'resolve_address', 'config', 'unit_get', @@ -15,17 +12,21 @@ TO_PATCH = [ 'add_bridge_port', 'service_running', 'service_start', + 'service_restart', 'get_host_ip', - 'get_nic_hwaddr', - 'list_nics', ] +def fake_context(settings): + def outer(): + def inner(): + return settings + return inner + return outer class OVSPluginContextTest(CharmTestCase): def setUp(self): super(OVSPluginContextTest, self).setUp(context, TO_PATCH) - self.relation_get.side_effect = self.test_relation.get self.config.side_effect = self.test_config.get self.test_config.set('debug', True) self.test_config.set('verbose', True) @@ -34,38 +35,57 @@ class OVSPluginContextTest(CharmTestCase): def tearDown(self): super(OVSPluginContextTest, self).tearDown() - def test_data_port_name(self): - self.test_config.set('data-port', 'em1') - self.assertEquals(context.OVSPluginContext().get_data_port(), 'em1') + @patch('charmhelpers.contrib.openstack.context.config') + @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' + 'resolve_ports') + def test_data_port_name(self, mock_resolve_ports, config): + self.test_config.set('data-port', 'br-data:em1') + config.side_effect = self.test_config.get + mock_resolve_ports.side_effect = lambda ports: ports + self.assertEquals(context.DataPortContext()(), + {'br-data': 'em1'}) - def test_data_port_mac(self): + @patch('charmhelpers.contrib.openstack.context.config') + @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.list_nics') + def test_data_port_mac(self, list_nics, get_nic_hwaddr, config): machine_machs = { 'em1': 'aa:aa:aa:aa:aa:aa', 'eth0': 'bb:bb:bb:bb:bb:bb', } absent_mac = "cc:cc:cc:cc:cc:cc" - config_macs = "%s %s" % (absent_mac, machine_machs['em1']) + config_macs = ("br-d1:%s br-d2:%s" % + (absent_mac, machine_machs['em1'])) self.test_config.set('data-port', config_macs) + config.side_effect = self.test_config.get + list_nics.return_value = machine_machs.keys() + get_nic_hwaddr.side_effect = lambda nic: machine_machs[nic] + self.assertEquals(context.DataPortContext()(), + {'br-d2': 'em1'}) - def get_hwaddr(eth): - return machine_machs[eth] - self.get_nic_hwaddr.side_effect = get_hwaddr - self.list_nics.return_value = machine_machs.keys() - self.assertEquals(context.OVSPluginContext().get_data_port(), 'em1') + @patch('charmhelpers.contrib.openstack.context.config') + @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' + 'resolve_ports') + def test_ensure_bridge_data_port_present(self, mock_resolve_ports, config): + self.test_config.set('data-port', 'br-data:em1') + self.test_config.set('bridge-mappings', 'phybr1:br-data') + config.side_effect = self.test_config.get - @patch.object(context.OVSPluginContext, 'get_data_port') - def test_ensure_bridge_data_port_present(self, get_data_port): def add_port(bridge, port, promisc): + if bridge == 'br-data' and port == 'em1' and promisc is True: self.bridge_added = True return self.bridge_added = False - get_data_port.return_value = 'em1' + mock_resolve_ports.side_effect = lambda ports: ports self.add_bridge_port.side_effect = add_port context.OVSPluginContext()._ensure_bridge() self.assertEquals(self.bridge_added, True) + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') @patch.object(charmhelpers.contrib.openstack.context, 'config') @patch.object(charmhelpers.contrib.openstack.context, 'unit_get') @patch.object(charmhelpers.contrib.openstack.context, 'is_clustered') @@ -77,7 +97,7 @@ class OVSPluginContextTest(CharmTestCase): @patch.object(charmhelpers.contrib.openstack.context, 'unit_private_ip') def test_neutroncc_context_api_rel(self, _unit_priv_ip, _npa, _ens_pkgs, _save_ff, _https, _is_clus, _unit_get, - _config): + _config, _runits, _rids, _rget): def mock_npa(plugin, section, manager): if section == "driver": return "neutron.randomdriver" @@ -88,13 +108,16 @@ class OVSPluginContextTest(CharmTestCase): _unit_get.return_value = '127.0.0.13' _unit_priv_ip.return_value = '127.0.0.14' _is_clus.return_value = False - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'l2-population': 'True', - 'enable-dvr': 'True', - 'overlay-network-type': 'gre', - }) + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'l2-population': 'True', + 'network-device-mtu': 1500, + 'overlay-network-type': 'gre', + 'enable-dvr': 'True', + } + _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() @@ -104,6 +127,8 @@ class OVSPluginContextTest(CharmTestCase): 'distributed_routing': True, 'verbose': True, 'local_ip': '127.0.0.15', + 'network_device_mtu': 1500, + 'veth_mtu': 1500, 'config': 'neutron.randomconfig', 'use_syslog': True, 'network_manager': 'neutron', @@ -113,10 +138,16 @@ class OVSPluginContextTest(CharmTestCase): 'neutron_url': 'https://127.0.0.13:9696', 'l2_population': True, 'overlay_network_type': 'gre', + 'network_providers': 'physnet1', + 'bridge_mappings': 'physnet1:br-data', + 'vlan_ranges': 'physnet1:1000:2000', } self.assertEquals(expect, napi_ctxt()) self.service_start.assertCalled() + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') @patch.object(charmhelpers.contrib.openstack.context, 'config') @patch.object(charmhelpers.contrib.openstack.context, 'unit_get') @patch.object(charmhelpers.contrib.openstack.context, 'is_clustered') @@ -131,24 +162,29 @@ class OVSPluginContextTest(CharmTestCase): _ens_pkgs, _save_ff, _https, _is_clus, _unit_get, - _config): + _config, _runits, + _rids, _rget): def mock_npa(plugin, section, manager): if section == "driver": return "neutron.randomdriver" if section == "config": return "neutron.randomconfig" + _npa.side_effect = mock_npa _config.return_value = 'ovs' _unit_get.return_value = '127.0.0.13' _unit_priv_ip.return_value = '127.0.0.14' _is_clus.return_value = False self.test_config.set('disable-security-groups', True) - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'l2-population': 'True', - 'overlay-network-type': 'gre', - }) + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'l2-population': 'True', + 'network-device-mtu': 1500, + 'overlay-network-type': 'gre', + } + _rget.side_effect = lambda *args, **kwargs: rdata self.get_host_ip.return_value = '127.0.0.15' self.service_running.return_value = False napi_ctxt = context.OVSPluginContext() @@ -158,6 +194,8 @@ class OVSPluginContextTest(CharmTestCase): 'neutron_security_groups': False, 'verbose': True, 'local_ip': '127.0.0.15', + 'veth_mtu': 1500, + 'network_device_mtu': 1500, 'config': 'neutron.randomconfig', 'use_syslog': True, 'network_manager': 'neutron', @@ -167,6 +205,9 @@ class OVSPluginContextTest(CharmTestCase): 'neutron_url': 'https://127.0.0.13:9696', 'l2_population': True, 'overlay_network_type': 'gre', + 'network_providers': 'physnet1', + 'bridge_mappings': 'physnet1:br-data', + 'vlan_ranges': 'physnet1:1000:2000', } self.assertEquals(expect, napi_ctxt()) self.service_start.assertCalled() @@ -176,61 +217,44 @@ class L3AgentContextTest(CharmTestCase): def setUp(self): super(L3AgentContextTest, self).setUp(context, TO_PATCH) - self.relation_get.side_effect = self.test_relation.get self.config.side_effect = self.test_config.get def tearDown(self): super(L3AgentContextTest, self).tearDown() - def test_dvr_enabled(self): - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'enable-dvr': 'True', - 'l2-population': 'True', - 'overlay-network-type': 'vxlan'}) + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') + def test_dvr_enabled(self, _runits, _rids, _rget): + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'enable-dvr': 'True', + 'l2-population': 'True', + 'overlay-network-type': 'vxlan', + 'network-device-mtu': 1500, + } + _rget.side_effect = lambda *args, **kwargs: rdata self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'dvr'}) - def test_dvr_disabled(self): - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'neutron-security-groups': 'True', - 'enable-dvr': 'False', - 'l2-population': 'True', - 'overlay-network-type': 'vxlan'}) + @patch.object(charmhelpers.contrib.openstack.context, 'relation_get') + @patch.object(charmhelpers.contrib.openstack.context, 'relation_ids') + @patch.object(charmhelpers.contrib.openstack.context, 'related_units') + def test_dvr_disabled(self, _runits, _rids, _rget): + _runits.return_value = ['unit1'] + _rids.return_value = ['rid2'] + rdata = { + 'neutron-security-groups': 'True', + 'enable-dvr': 'False', + 'l2-population': 'True', + 'overlay-network-type': 'vxlan', + 'network-device-mtu': 1500, + } + _rget.side_effect = lambda *args, **kwargs: rdata self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'legacy'}) -class NetworkServiceContext(CharmTestCase): - - def setUp(self): - super(NetworkServiceContext, self).setUp(context, TO_PATCH) - self.relation_get.side_effect = self.test_relation.get - self.config.side_effect = self.test_config.get - - def tearDown(self): - super(NetworkServiceContext, self).tearDown() - - def test_network_svc_ctxt(self): - self.related_units.return_value = ['unit1'] - self.relation_ids.return_value = ['rid2'] - self.test_relation.set({'service_protocol': 'http', - 'keystone_host': '10.0.0.10', - 'service_port': '8080', - 'region': 'region1', - 'service_tenant': 'tenant', - 'service_username': 'bob', - 'service_password': 'reallyhardpass'}) - self.assertEquals(context.NetworkServiceContext()(), - {'service_protocol': 'http', - 'keystone_host': '10.0.0.10', - 'service_port': '8080', - 'region': 'region1', - 'service_tenant': 'tenant', - 'service_username': 'bob', - 'service_password': 'reallyhardpass'}) - - class DVRSharedSecretContext(CharmTestCase): def setUp(self): @@ -238,6 +262,7 @@ class DVRSharedSecretContext(CharmTestCase): TO_PATCH) self.config.side_effect = self.test_config.get + @patch('os.path') @patch('uuid.uuid4') def test_secret_created_stored(self, _uuid4, _path): @@ -260,20 +285,21 @@ class DVRSharedSecretContext(CharmTestCase): _open.assert_called_with( context.SHARED_SECRET.format('quantum'), 'r') - @patch.object(context, 'use_dvr') + @patch.object(context, 'NeutronAPIContext') @patch.object(context, 'get_shared_secret') - def test_shared_secretcontext_dvr(self, _shared_secret, _use_dvr): + def test_shared_secretcontext_dvr(self, _shared_secret, _NeutronAPIContext): + _NeutronAPIContext.side_effect = fake_context({'enable_dvr': True}) _shared_secret.return_value = 'secret_thing' - _use_dvr.return_value = True + #_use_dvr.return_value = True self.resolve_address.return_value = '10.0.0.10' self.assertEquals(context.DVRSharedSecretContext()(), {'shared_secret': 'secret_thing', 'local_ip': '10.0.0.10'}) - @patch.object(context, 'use_dvr') + @patch.object(context, 'NeutronAPIContext') @patch.object(context, 'get_shared_secret') - def test_shared_secretcontext_nodvr(self, _shared_secret, _use_dvr): + def test_shared_secretcontext_nodvr(self, _shared_secret, _NeutronAPIContext): + _NeutronAPIContext.side_effect = fake_context({'enable_dvr': False}) _shared_secret.return_value = 'secret_thing' - _use_dvr.return_value = False self.resolve_address.return_value = '10.0.0.10' self.assertEquals(context.DVRSharedSecretContext()(), {}) diff --git a/unit_tests/test_neutron_ovs_hooks.py b/unit_tests/test_neutron_ovs_hooks.py index dfa7d29d..b167c886 100644 --- a/unit_tests/test_neutron_ovs_hooks.py +++ b/unit_tests/test_neutron_ovs_hooks.py @@ -59,16 +59,12 @@ class NeutronOVSHooksTests(CharmTestCase): call(_pkgs, fatal=True), ]) - @patch.object(neutron_ovs_context, 'use_dvr') - def test_config_changed(self, _use_dvr): - _use_dvr.return_value = False + def test_config_changed(self): self._call_hook('config-changed') self.assertTrue(self.CONFIGS.write_all.called) self.configure_ovs.assert_called_with() - @patch.object(neutron_ovs_context, 'use_dvr') - def test_config_changed_dvr(self, _use_dvr): - _use_dvr.return_value = True + def test_config_changed_dvr(self): self.determine_dvr_packages.return_value = ['dvr'] self._call_hook('config-changed') self.apt_update.assert_called_with() @@ -79,9 +75,7 @@ class NeutronOVSHooksTests(CharmTestCase): self.configure_ovs.assert_called_with() @patch.object(hooks, 'neutron_plugin_joined') - @patch.object(neutron_ovs_context, 'use_dvr') - def test_neutron_plugin_api(self, _use_dvr, _plugin_joined): - _use_dvr.return_value = False + def test_neutron_plugin_api(self, _plugin_joined): self.relation_ids.return_value = ['rid'] self._call_hook('neutron-plugin-api-relation-changed') self.configure_ovs.assert_called_with() diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 6ed7f357..3ad82284 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -61,10 +61,10 @@ class TestNeutronOVSUtils(CharmTestCase): # Reset cached cache hookenv.cache = {} + @patch.object(nutils, 'use_dvr') @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release') @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package') - @patch.object(neutron_ovs_context, 'use_dvr') - def test_determine_packages(self, _use_dvr, _head_pkgs, _os_rel): + def test_determine_packages(self, _head_pkgs, _os_rel, _use_dvr): _use_dvr.return_value = False _os_rel.return_value = 'trusty' _head_pkgs.return_value = head_pkg @@ -72,7 +72,7 @@ class TestNeutronOVSUtils(CharmTestCase): expect = [['neutron-plugin-openvswitch-agent'], [head_pkg]] self.assertItemsEqual(pkg_list, expect) - @patch.object(neutron_ovs_context, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_register_configs(self, _use_dvr): class _mock_OSConfigRenderer(): def __init__(self, templates_dir=None, openstack_release=None): @@ -88,10 +88,11 @@ class TestNeutronOVSUtils(CharmTestCase): templating.OSConfigRenderer.side_effect = _mock_OSConfigRenderer _regconfs = nutils.register_configs() confs = ['/etc/neutron/neutron.conf', - '/etc/neutron/plugins/ml2/ml2_conf.ini'] + '/etc/neutron/plugins/ml2/ml2_conf.ini', + '/etc/init/os-charm-phy-nic-mtu.conf'] self.assertItemsEqual(_regconfs.configs, confs) - @patch.object(neutron_ovs_context, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_resource_map(self, _use_dvr): _use_dvr.return_value = False _map = nutils.resource_map() @@ -100,7 +101,7 @@ class TestNeutronOVSUtils(CharmTestCase): [self.assertIn(q_conf, _map.keys()) for q_conf in confs] self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) - @patch.object(neutron_ovs_context, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_resource_map_dvr(self, _use_dvr): _use_dvr.return_value = True _map = nutils.resource_map() @@ -110,7 +111,7 @@ class TestNeutronOVSUtils(CharmTestCase): [self.assertIn(q_conf, _map.keys()) for q_conf in confs] self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs) - @patch.object(neutron_ovs_context, 'use_dvr') + @patch.object(nutils, 'use_dvr') def test_restart_map(self, _use_dvr): _use_dvr.return_value = False _restart_map = nutils.restart_map() @@ -118,8 +119,9 @@ class TestNeutronOVSUtils(CharmTestCase): expect = OrderedDict([ (nutils.NEUTRON_CONF, ['neutron-plugin-openvswitch-agent']), (ML2CONF, ['neutron-plugin-openvswitch-agent']), + (nutils.PHY_NIC_MTU_CONF, ['os-charm-phy-nic-mtu']) ]) - self.assertTrue(len(expect) == len(_restart_map)) + self.assertEqual(expect, _restart_map) for item in _restart_map: self.assertTrue(item in _restart_map) self.assertTrue(expect[item] == _restart_map[item])