openstack/charm-neutron-gateway
Code Issues Proposed changes

[gnuoy,r=james-page,t=james-page] Add support for specifying external port by mac address

Browse Source 
Also resync charm-helpers
This commit is contained in:
James Page 2014-03-14 13:24:39 +00:00
commit 7faeded915
14 changed files with 303 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -54,6 +54,22 @@ See upstream [Neutron multi extnet](http://docs.openstack.org/trunk/config-refer
Configuration Options
---------------------
External Port Configuration
===========================
If the port to be used for external traffic is consistent accross all physical
servers then is can be specified by simply setting ext-port to the nic id:
quantum-gateway:
ext-port: eth2
However, if it varies between hosts then the mac addresses of the external
nics for each host can be passed as a space seperated list:
quantum-gateway:
ext-port: <MAC ext port host 1> <MAC ext port host 2> <MAC ext port host 3>
Multiple Floating Pools
=======================

8
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@ -126,17 +126,17 @@ def determine_api_port(public_port):
return public_port - (i * 10)
def determine_haproxy_port(public_port):
def determine_apache_port(public_port):
'''
Description: Determine correct proxy listening port based on public IP +
existence of HTTPS reverse proxy.
Description: Determine correct apache listening port based on public IP +
state of the cluster.
public_port: int: standard public port for given service
returns: int: the correct listening port for the HAProxy service
'''
i = 0
if https():
if len(peer_units()) > 0 or is_clustered():
i += 1
return public_port - (i * 10)

137
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -23,15 +23,13 @@ from charmhelpers.core.hookenv import (
unit_get,
unit_private_ip,
ERROR,
WARNING,
)
from charmhelpers.contrib.hahelpers.cluster import (
determine_apache_port,
determine_api_port,
determine_haproxy_port,
https,
is_clustered,
peer_units,
is_clustered
)
from charmhelpers.contrib.hahelpers.apache import (
@ -68,6 +66,43 @@ def context_complete(ctxt):
return True
def config_flags_parser(config_flags):
if config_flags.find('==') >= 0:
log("config_flags is not in expected format (key=value)",
level=ERROR)
raise OSContextError
# strip the following from each value.
post_strippers = ' ,'
# we strip any leading/trailing '=' or ' ' from the string then
# split on '='.
split = config_flags.strip(' =').split('=')
limit = len(split)
flags = {}
for i in xrange(0, limit - 1):
current = split[i]
next = split[i + 1]
vindex = next.rfind(',')
if (i == limit - 2) or (vindex < 0):
value = next
else:
value = next[:vindex]
if i == 0:
key = current
else:
# if this not the first entry, expect an embedded key.
index = current.rfind(',')
if index < 0:
log("invalid config value(s) at index %s" % (i),
level=ERROR)
raise OSContextError
key = current[index + 1:]
# Add to collection.
flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
return flags
class OSContextGenerator(object):
interfaces = []
@ -182,10 +217,17 @@ class AMQPContext(OSContextGenerator):
# Sufficient information found = break out!
break
# Used for active/active rabbitmq >= grizzly
ctxt['rabbitmq_hosts'] = []
for unit in related_units(rid):
ctxt['rabbitmq_hosts'].append(relation_get('private-address',
rid=rid, unit=unit))
if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \
len(related_units(rid)) > 1:
if relation_get('ha_queues'):
ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues')
else:
ctxt['rabbitmq_ha_queues'] = False
rabbitmq_hosts = []
for unit in related_units(rid):
rabbitmq_hosts.append(relation_get('private-address',
rid=rid, unit=unit))
ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
if not context_complete(ctxt):
return {}
else:
@ -199,10 +241,13 @@ class CephContext(OSContextGenerator):
'''This generates context for /etc/ceph/ceph.conf templates'''
if not relation_ids('ceph'):
return {}
log('Generating template context for ceph')
mon_hosts = []
auth = None
key = None
use_syslog = str(config('use-syslog')).lower()
for rid in relation_ids('ceph'):
for unit in related_units(rid):
mon_hosts.append(relation_get('private-address', rid=rid,
@ -214,6 +259,7 @@ class CephContext(OSContextGenerator):
'mon_hosts': ' '.join(mon_hosts),
'auth': auth,
'key': key,
'use_syslog': use_syslog
}
if not os.path.isdir('/etc/ceph'):
@ -286,6 +332,7 @@ class ImageServiceContext(OSContextGenerator):
class ApacheSSLContext(OSContextGenerator):
"""
Generates a context for an apache vhost configuration that configures
HTTPS reverse proxying for one or many endpoints. Generated context
@ -341,17 +388,15 @@ class ApacheSSLContext(OSContextGenerator):
'private_address': unit_get('private-address'),
'endpoints': []
}
for ext_port in self.external_ports:
if peer_units() or is_clustered():
int_port = determine_haproxy_port(ext_port)
else:
int_port = determine_api_port(ext_port)
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port)
int_port = determine_api_port(api_port)
portmap = (int(ext_port), int(int_port))
ctxt['endpoints'].append(portmap)
return ctxt
class NeutronContext(object):
class NeutronContext(OSContextGenerator):
interfaces = []
@property
@ -412,6 +457,22 @@ class NeutronContext(object):
return nvp_ctxt
def neutron_ctxt(self):
if https():
proto = 'https'
else:
proto = 'http'
if is_clustered():
host = config('vip')
else:
host = unit_get('private-address')
url = '%s://%s:%s' % (proto, host, '9696')
ctxt = {
'network_manager': self.network_manager,
'neutron_url': url,
}
return ctxt
def __call__(self):
self._ensure_packages()
@ -421,40 +482,44 @@ class NeutronContext(object):
if not self.plugin:
return {}
ctxt = {'network_manager': self.network_manager}
ctxt = self.neutron_ctxt()
if self.plugin == 'ovs':
ctxt.update(self.ovs_ctxt())
elif self.plugin == 'nvp':
ctxt.update(self.nvp_ctxt())
alchemy_flags = config('neutron-alchemy-flags')
if alchemy_flags:
flags = config_flags_parser(alchemy_flags)
ctxt['neutron_alchemy_flags'] = flags
self._save_flag_file()
return ctxt
class OSConfigFlagContext(OSContextGenerator):
'''
Responsible adding user-defined config-flags in charm config to a
to a template context.
'''
"""
Responsible for adding user-defined config-flags in charm config to a
template context.
NOTE: the value of config-flags may be a comma-separated list of
key=value pairs and some Openstack config files support
comma-separated lists as values.
"""
def __call__(self):
config_flags = config('config-flags')
if not config_flags or config_flags in ['None', '']:
if not config_flags:
return {}
config_flags = config_flags.split(',')
flags = {}
for flag in config_flags:
if '=' not in flag:
log('Improperly formatted config-flag, expected k=v '
'got %s' % flag, level=WARNING)
continue
k, v = flag.split('=')
flags[k.strip()] = v
ctxt = {'user_config_flags': flags}
return ctxt
flags = config_flags_parser(config_flags)
return {'user_config_flags': flags}
class SubordinateConfigContext(OSContextGenerator):
"""
Responsible for inspecting relations to subordinates that
may be exporting required config via a json blob.
@ -495,6 +560,7 @@ class SubordinateConfigContext(OSContextGenerator):
}
"""
def __init__(self, service, config_file, interface):
"""
:param service : Service name key to query in any subordinate
@ -539,3 +605,12 @@ class SubordinateConfigContext(OSContextGenerator):
ctxt['sections'] = {}
return ctxt
class SyslogContext(OSContextGenerator):
def __call__(self):
ctxt = {
'use_syslog': config('use-syslog')
}
return ctxt

20
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -41,6 +41,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('quantal', 'folsom'),
('raring', 'grizzly'),
('saucy', 'havana'),
('trusty', 'icehouse')
])
@ -201,7 +202,7 @@ def os_release(package, base='essex'):
def import_key(keyid):
cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \
cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \
"--recv-keys %s" % keyid
try:
subprocess.check_call(cmd.split(' '))
@ -260,6 +261,9 @@ def configure_installation_source(rel):
'havana': 'precise-updates/havana',
'havana/updates': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'icehouse': 'precise-updates/icehouse',
'icehouse/updates': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
}
try:
@ -411,7 +415,7 @@ def get_host_ip(hostname):
return ns_query(hostname)
def get_hostname(address):
def get_hostname(address, fqdn=True):
"""
Resolves hostname for given IP, or returns the input
if it is already a hostname.
@ -430,7 +434,11 @@ def get_hostname(address):
if not result:
return None
# strip trailing .
if result.endswith('.'):
return result[:-1]
return result
if fqdn:
# strip trailing .
if result.endswith('.'):
return result[:-1]
else:
return result
else:
return result.split('.')[0]

8
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -49,6 +49,9 @@ CEPH_CONF = """[global]
auth supported = {auth}
keyring = {keyring}
mon host = {mon_hosts}
log to syslog = {use_syslog}
err to syslog = {use_syslog}
clog to syslog = {use_syslog}
"""
@ -194,7 +197,7 @@ def get_ceph_nodes():
return hosts
def configure(service, key, auth):
def configure(service, key, auth, use_syslog):
''' Perform basic configuration of Ceph '''
create_keyring(service, key)
create_key_file(service, key)
@ -202,7 +205,8 @@ def configure(service, key, auth):
with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
ceph_conf.write(CEPH_CONF.format(auth=auth,
keyring=_keyring_path(service),
mon_hosts=",".join(map(str, hosts))))
mon_hosts=",".join(map(str, hosts)),
use_syslog=use_syslog))
modprobe('rbd')

2
hooks/charmhelpers/contrib/storage/linux/utils.py
View File

@ -22,4 +22,4 @@ def zap_disk(block_device):
:param block_device: str: Full path of block device to clean.
'''
check_call(['sgdisk', '--zap-all', block_device])
check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])

6
hooks/charmhelpers/core/hookenv.py
View File

@ -8,6 +8,7 @@ import os
import json
import yaml
import subprocess
import sys
import UserDict
from subprocess import CalledProcessError
@ -149,6 +150,11 @@ def service_name():
return local_unit().split('/')[0]
def hook_name():
"""The name of the currently executing hook"""
return os.path.basename(sys.argv[0])
@cached
def config(scope=None):
"""Juju charm configuration"""

56
hooks/charmhelpers/core/host.py
View File

@ -194,7 +194,7 @@ def file_hash(path):
return None
def restart_on_change(restart_map):
def restart_on_change(restart_map, stopstart=False):
"""Restart services based on configuration files changing
This function is used a decorator, for example
@ -219,8 +219,14 @@ def restart_on_change(restart_map):
for path in restart_map:
if checksums[path] != file_hash(path):
restarts += restart_map[path]
for service_name in list(OrderedDict.fromkeys(restarts)):
service('restart', service_name)
services_list = list(OrderedDict.fromkeys(restarts))
if not stopstart:
for service_name in services_list:
service('restart', service_name)
else:
for action in ['stop', 'start']:
for service_name in services_list:
service(action, service_name)
return wrapped_f
return wrap
@ -245,3 +251,47 @@ def pwgen(length=None):
random_chars = [
random.choice(alphanumeric_chars) for _ in range(length)]
return(''.join(random_chars))
def list_nics(nic_type):
'''Return a list of nics of given type(s)'''
if isinstance(nic_type, basestring):
int_types = [nic_type]
else:
int_types = nic_type
interfaces = []
for int_type in int_types:
cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
ip_output = subprocess.check_output(cmd).split('\n')
ip_output = (line for line in ip_output if line)
for line in ip_output:
if line.split()[1].startswith(int_type):
interfaces.append(line.split()[1].replace(":", ""))
return interfaces
def set_nic_mtu(nic, mtu):
'''Set MTU on a network interface'''
cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
subprocess.check_call(cmd)
def get_nic_mtu(nic):
cmd = ['ip', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd).split('\n')
mtu = ""
for line in ip_output:
words = line.split()
if 'mtu' in words:
mtu = words[words.index("mtu") + 1]
return mtu
def get_nic_hwaddr(nic):
cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
ip_output = subprocess.check_output(cmd)
hwaddr = ""
words = ip_output.split()
if 'link/ether' in words:
hwaddr = words[words.index('link/ether') + 1]
return hwaddr

52
hooks/charmhelpers/fetch/__init__.py
View File

@ -13,6 +13,7 @@ from charmhelpers.core.hookenv import (
log,
)
import apt_pkg
import os
CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@ -43,8 +44,16 @@ CLOUD_ARCHIVE_POCKETS = {
'precise-havana/updates': 'precise-updates/havana',
'precise-updates/havana': 'precise-updates/havana',
'havana/proposed': 'precise-proposed/havana',
'precies-havana/proposed': 'precise-proposed/havana',
'precise-havana/proposed': 'precise-proposed/havana',
'precise-proposed/havana': 'precise-proposed/havana',
# Icehouse
'icehouse': 'precise-updates/icehouse',
'precise-icehouse': 'precise-updates/icehouse',
'precise-icehouse/updates': 'precise-updates/icehouse',
'precise-updates/icehouse': 'precise-updates/icehouse',
'icehouse/proposed': 'precise-proposed/icehouse',
'precise-icehouse/proposed': 'precise-proposed/icehouse',
'precise-proposed/icehouse': 'precise-proposed/icehouse',
}
@ -66,8 +75,10 @@ def filter_installed_packages(packages):
def apt_install(packages, options=None, fatal=False):
"""Install one or more packages"""
options = options or []
cmd = ['apt-get', '-y']
if options is None:
options = ['--option=Dpkg::Options::=--force-confold']
cmd = ['apt-get', '--assume-yes']
cmd.extend(options)
cmd.append('install')
if isinstance(packages, basestring):
@ -76,10 +87,14 @@ def apt_install(packages, options=None, fatal=False):
cmd.extend(packages)
log("Installing {} with options: {}".format(packages,
options))
env = os.environ.copy()
if 'DEBIAN_FRONTEND' not in env:
env['DEBIAN_FRONTEND'] = 'noninteractive'
if fatal:
subprocess.check_call(cmd)
subprocess.check_call(cmd, env=env)
else:
subprocess.call(cmd)
subprocess.call(cmd, env=env)
def apt_update(fatal=False):
@ -93,7 +108,7 @@ def apt_update(fatal=False):
def apt_purge(packages, fatal=False):
"""Purge one or more packages"""
cmd = ['apt-get', '-y', 'purge']
cmd = ['apt-get', '--assume-yes', 'purge']
if isinstance(packages, basestring):
cmd.append(packages)
else:
@ -120,17 +135,23 @@ def apt_hold(packages, fatal=False):
def add_source(source, key=None):
if source is None:
log('Source is not present. Skipping')
return
if (source.startswith('ppa:') or
source.startswith('http:') or
source.startswith('http') or
source.startswith('deb ') or
source.startswith('cloud-archive:')):
source.startswith('cloud-archive:')):
subprocess.check_call(['add-apt-repository', '--yes', source])
elif source.startswith('cloud:'):
apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
fatal=True)
pocket = source.split(':')[-1]
if pocket not in CLOUD_ARCHIVE_POCKETS:
raise SourceConfigError('Unsupported cloud: source option %s' % pocket)
raise SourceConfigError(
'Unsupported cloud: source option %s' %
pocket)
actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
apt.write(CLOUD_ARCHIVE.format(actual_pocket))
@ -139,7 +160,9 @@ def add_source(source, key=None):
with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
apt.write(PROPOSED_POCKET.format(release))
if key:
subprocess.check_call(['apt-key', 'import', key])
subprocess.check_call(['apt-key', 'adv', '--keyserver',
'keyserver.ubuntu.com', '--recv',
key])
class SourceConfigError(Exception):
@ -220,7 +243,9 @@ def install_from_config(config_var_name):
class BaseFetchHandler(object):
"""Base class for FetchHandler implementations in fetch plugins"""
def can_handle(self, source):
"""Returns True if the source can be handled. Otherwise returns
a string explaining why it cannot"""
@ -248,10 +273,13 @@ def plugins(fetch_handlers=None):
for handler_name in fetch_handlers:
package, classname = handler_name.rsplit('.', 1)
try:
handler_class = getattr(importlib.import_module(package), classname)
handler_class = getattr(
importlib.import_module(package),
classname)
plugin_list.append(handler_class())
except (ImportError, AttributeError):
# Skip missing plugins so that they can be ommitted from
# installation if desired
log("FetchHandler {} not found, skipping plugin".format(handler_name))
log("FetchHandler {} not found, skipping plugin".format(
handler_name))
return plugin_list

21
hooks/quantum_contexts.py
View File

@ -2,6 +2,10 @@
import os
import uuid
import socket
from charmhelpers.core.host import (
list_nics,
get_nic_hwaddr
)
from charmhelpers.core.hookenv import (
config,
relation_ids,
@ -23,6 +27,7 @@ from charmhelpers.contrib.openstack.utils import (
from charmhelpers.contrib.hahelpers.cluster import(
eligible_leader
)
import re
DB_USER = "quantum"
QUANTUM_DB = "quantum"
@ -121,10 +126,20 @@ class L3AgentContext(OSContextGenerator):
class ExternalPortContext(OSContextGenerator):
def __call__(self):
if config('ext-port'):
return {"ext_port": config('ext-port')}
else:
if not config('ext-port'):
return None
hwaddrs = {}
for nic in list_nics(['eth', 'bond']):
hwaddrs[get_nic_hwaddr(nic)] = nic
mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
for entry in config('ext-port').split():
entry = entry.strip()
if re.match(mac_regex, entry):
if entry in hwaddrs:
return {"ext_port": hwaddrs[entry]}
else:
return {"ext_port": entry}
return None
class QuantumGatewayContext(OSContextGenerator):

6
hooks/quantum_utils.py
View File

@ -434,6 +434,6 @@ def configure_ovs():
full_restart()
add_bridge(INT_BRIDGE)
add_bridge(EXT_BRIDGE)
ext_port = config('ext-port')
if ext_port:
add_bridge_port(EXT_BRIDGE, ext_port)
ext_port_ctx = ExternalPortContext()()
if ext_port_ctx is not None and ext_port_ctx['ext_port']:
add_bridge_port(EXT_BRIDGE, ext_port_ctx['ext_port'])

2
revision
View File

@ -1 +1 @@
60
63

25
unit_tests/test_quantum_contexts.py
View File

@ -16,6 +16,8 @@ TO_PATCH = [
'apt_install',
'get_os_codename_install_source',
'eligible_leader',
'list_nics',
'get_nic_hwaddr',
]
@ -136,11 +138,32 @@ class TestExternalPortContext(CharmTestCase):
self.assertEquals(quantum_contexts.ExternalPortContext()(),
None)
def test_ext_port(self):
def test_ext_port_eth(self):
self.config.return_value = 'eth1010'
self.assertEquals(quantum_contexts.ExternalPortContext()(),
{'ext_port': 'eth1010'})
def test_ext_port_mac(self):
machine_macs = {
'eth0': 'fe:c5:ce:8e:2b:00',
'eth1': 'fe:c5:ce:8e:2b:01',
'eth2': 'fe:c5:ce:8e:2b:02',
'eth3': 'fe:c5:ce:8e:2b:03',
}
absent_macs = "aa:a5:ae:ae:ab:a4 "
config_macs = absent_macs + " " + machine_macs['eth2']
def get_hwaddr(arg):
return machine_macs[arg]
self.config.return_value = config_macs
self.list_nics.return_value = machine_macs.keys()
self.get_nic_hwaddr.side_effect = get_hwaddr
self.assertEquals(quantum_contexts.ExternalPortContext()(),
{'ext_port': 'eth2'})
self.config.return_value = absent_macs
self.assertEquals(quantum_contexts.ExternalPortContext()(),
None)
class TestL3AgentContext(CharmTestCase):
def setUp(self):

11
unit_tests/test_quantum_utils.py
View File

@ -34,6 +34,7 @@ TO_PATCH = [
'full_restart',
'service_running',
'NetworkServiceContext',
'ExternalPortContext',
'unit_private_ip',
'relations_of_type',
'service_stop',
@ -100,6 +101,8 @@ class TestQuantumUtils(CharmTestCase):
self.config.side_effect = self.test_config.get
self.test_config.set('plugin', 'ovs')
self.test_config.set('ext-port', 'eth0')
self.ExternalPortContext.return_value = \
DummyExternalPortContext(return_value={'ext_port': 'eth0'})
quantum_utils.configure_ovs()
self.add_bridge.assert_has_calls([
call('br-int'),
@ -264,6 +267,14 @@ class DummyNetworkServiceContext():
def __call__(self):
return self.return_value
class DummyExternalPortContext():
def __init__(self, return_value):
self.return_value = return_value
def __call__(self):
return self.return_value
agents_all_alive = {
'DHCP Agent': {
'agents': [