[gnuoy,r=james-page] Add support for local DHCP and Metadata agents

This change allows nova-compute nodes to also run Neutron DHCP and Metadata
agents, allowing deploying without the neutron-gateway charm in VLAN and
flat networking configurations.

Only useful if l3 routing, vpnaas, fwaas and lbaas services are not required.

config.yaml

@@ -98,4 +98,11 @@ options:
       traffic to the external public network. Valid values are either MAC
       addresses (in which case only MAC addresses for interfaces without an IP
       address already assigned will be used), or interfaces (eth0)
-
+  enable-local-dhcp-and-metadata:
+    type: boolean
+    default: false
+    description: |
+      Enable local Neutron DHCP and Metadata Agents. This is useful for deployments
+      which do not include a neutron-gateway (do not require l3, lbaas or vpnaas
+      services) and should only be used in-conjunction with flat or VLAN provider
+      networks configurations.

hooks/neutron_ovs_context.py

@@ -108,10 +108,11 @@ def get_shared_secret():
     return secret
 
 
-class DVRSharedSecretContext(OSContextGenerator):
+class SharedSecretContext(OSContextGenerator):
 
     def __call__(self):
-        if NeutronAPIContext()()['enable_dvr']:
+        if NeutronAPIContext()()['enable_dvr'] or \
+                config('enable-local-dhcp-and-metadata'):
             ctxt = {
                 'shared_secret': get_shared_secret(),
                 'local_ip': resolve_address(),

hooks/neutron_ovs_hooks.py

@@ -20,25 +20,24 @@ from charmhelpers.core.host import (
     restart_on_change
 )
 
-from charmhelpers.fetch import (
-    apt_install, apt_update, apt_purge
-)
-
 from charmhelpers.contrib.openstack.utils import (
     os_requires_version,
 )
 
 from neutron_ovs_utils import (
+    DHCP_PACKAGES,
     DVR_PACKAGES,
     configure_ovs,
-    determine_packages,
     git_install,
     get_topics,
-    determine_dvr_packages,
     get_shared_secret,
     register_configs,
     restart_map,
     use_dvr,
+    enable_nova_metadata,
+    enable_local_dhcp,
+    install_packages,
+    purge_packages,
 )
 
 hooks = Hooks()
@@ -47,11 +46,7 @@ CONFIGS = register_configs()
 
 @hooks.hook()
 def install():
-    apt_update()
+    install_packages()
-    pkgs = determine_packages()
-    for pkg in pkgs:
-        apt_install(pkg, fatal=True)
-
     git_install(config('openstack-origin-git'))
 
 
@@ -59,10 +54,7 @@ def install():
 @hooks.hook('config-changed')
 @restart_on_change(restart_map())
 def config_changed():
-    if determine_dvr_packages():
+    install_packages()
-        apt_update()
-        apt_install(determine_dvr_packages(), fatal=True)
-
     if git_install_requested():
         if config_value_changed('openstack-origin-git'):
             git_install(config('openstack-origin-git'))
@@ -71,16 +63,17 @@ def config_changed():
     CONFIGS.write_all()
     for rid in relation_ids('zeromq-configuration'):
         zeromq_configuration_relation_joined(rid)
+    for rid in relation_ids('neutron-plugin'):
+        neutron_plugin_joined(relation_id=rid)
 
 
 @hooks.hook('neutron-plugin-api-relation-changed')
 @restart_on_change(restart_map())
 def neutron_plugin_api_changed():
     if use_dvr():
-        apt_update()
+        install_packages()
-        apt_install(DVR_PACKAGES, fatal=True)
     else:
-        apt_purge(DVR_PACKAGES, fatal=True)
+        purge_packages(DVR_PACKAGES)
     configure_ovs()
     CONFIGS.write_all()
     # If dvr setting has changed, need to pass that on
@@ -90,7 +83,11 @@ def neutron_plugin_api_changed():
 
 @hooks.hook('neutron-plugin-relation-joined')
 def neutron_plugin_joined(relation_id=None):
-    secret = get_shared_secret() if use_dvr() else None
+    if enable_local_dhcp():
+        install_packages()
+    else:
+        purge_packages(DHCP_PACKAGES)
+    secret = get_shared_secret() if enable_nova_metadata() else None
     rel_data = {
         'metadata-shared-secret': secret,
     }

hooks/neutron_ovs_utils.py

@@ -43,6 +43,13 @@ from charmhelpers.core.host import (
 
 from charmhelpers.core.templating import render
 
+from charmhelpers.fetch import (
+    apt_install,
+    apt_purge,
+    apt_update,
+    filter_installed_packages,
+)
+
 BASE_GIT_PACKAGES = [
     'libffi-dev',
     'libssl-dev',
@@ -66,6 +73,7 @@ GIT_PACKAGE_BLACKLIST = [
 ]
 
 NOVA_CONF_DIR = "/etc/nova"
+NEUTRON_DHCP_AGENT_CONF = "/etc/neutron/dhcp_agent.ini"
 NEUTRON_CONF_DIR = "/etc/neutron"
 NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR
 NEUTRON_DEFAULT = '/etc/default/neutron-server'
@@ -75,6 +83,7 @@ ML2_CONF = '%s/plugins/ml2/ml2_conf.ini' % NEUTRON_CONF_DIR
 EXT_PORT_CONF = '/etc/init/ext-port.conf'
 NEUTRON_METADATA_AGENT_CONF = "/etc/neutron/metadata_agent.ini"
 DVR_PACKAGES = ['neutron-l3-agent']
+DHCP_PACKAGES = ['neutron-metadata-agent', 'neutron-dhcp-agent']
 PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf'
 TEMPLATES = 'templates/'
 
@@ -95,6 +104,19 @@ BASE_RESOURCE_MAP = OrderedDict([
         'contexts': [context.PhyNICMTUContext()],
     }),
 ])
+METADATA_RESOURCE_MAP = OrderedDict([
+    (NEUTRON_METADATA_AGENT_CONF, {
+        'services': ['neutron-metadata-agent'],
+        'contexts': [neutron_ovs_context.SharedSecretContext(),
+                     neutron_ovs_context.APIIdentityServiceContext()],
+    }),
+])
+DHCP_RESOURCE_MAP = OrderedDict([
+    (NEUTRON_DHCP_AGENT_CONF, {
+        'services': ['neutron-dhcp-agent'],
+        'contexts': [],
+    }),
+])
 DVR_RESOURCE_MAP = OrderedDict([
     (NEUTRON_L3_AGENT_CONF, {
         'services': ['neutron-l3-agent'],
@@ -108,11 +130,6 @@ DVR_RESOURCE_MAP = OrderedDict([
         'services': ['neutron-l3-agent'],
         'contexts': [context.ExternalPortContext()],
     }),
-    (NEUTRON_METADATA_AGENT_CONF, {
-        'services': ['neutron-metadata-agent'],
-        'contexts': [neutron_ovs_context.DVRSharedSecretContext(),
-                     neutron_ovs_context.APIIdentityServiceContext()],
-    }),
 ])
 TEMPLATES = 'templates/'
 INT_BRIDGE = "br-int"
@@ -120,16 +137,30 @@ EXT_BRIDGE = "br-ex"
 DATA_BRIDGE = 'br-data'
 
 
-def determine_dvr_packages():
+def install_packages():
-    if not git_install_requested():
+    apt_update()
-        if use_dvr():
+    apt_install(filter_installed_packages(determine_packages()))
-            return DVR_PACKAGES
+
-    return []
+
+def purge_packages(pkg_list):
+    purge_pkgs = []
+    required_packages = determine_packages()
+    for pkg in pkg_list:
+        if pkg not in required_packages:
+            purge_pkgs.append(pkg)
+    if purge_pkgs:
+        apt_purge(purge_pkgs, fatal=True)
 
 
 def determine_packages():
-    pkgs = neutron_plugin_attribute('ovs', 'packages', 'neutron')
+    pkgs = []
-    pkgs.extend(determine_dvr_packages())
+    plugin_pkgs = neutron_plugin_attribute('ovs', 'packages', 'neutron')
+    for plugin_pkg in plugin_pkgs:
+        pkgs.extend(plugin_pkg)
+    if use_dvr():
+        pkgs.extend(DVR_PACKAGES)
+    if enable_local_dhcp():
+        pkgs.extend(DHCP_PACKAGES)
 
     if git_install_requested():
         pkgs.extend(BASE_GIT_PACKAGES)
@@ -158,8 +189,14 @@ def resource_map():
     resource_map = deepcopy(BASE_RESOURCE_MAP)
     if use_dvr():
         resource_map.update(DVR_RESOURCE_MAP)
+        resource_map.update(METADATA_RESOURCE_MAP)
         dvr_services = ['neutron-metadata-agent', 'neutron-l3-agent']
         resource_map[NEUTRON_CONF]['services'] += dvr_services
+    if enable_local_dhcp():
+        resource_map.update(METADATA_RESOURCE_MAP)
+        resource_map.update(DHCP_RESOURCE_MAP)
+        metadata_services = ['neutron-metadata-agent', 'neutron-dhcp-agent']
+        resource_map[NEUTRON_CONF]['services'] += metadata_services
     return resource_map
 
 
@@ -209,7 +246,7 @@ def configure_ovs():
 
 
 def get_shared_secret():
-    ctxt = neutron_ovs_context.DVRSharedSecretContext()()
+    ctxt = neutron_ovs_context.SharedSecretContext()()
     if 'shared_secret' in ctxt:
         return ctxt['shared_secret']
 
@@ -218,6 +255,14 @@ def use_dvr():
     return context.NeutronAPIContext()()['enable_dvr']
 
 
+def enable_nova_metadata():
+    return use_dvr() or enable_local_dhcp()
+
+
+def enable_local_dhcp():
+    return config('enable-local-dhcp-and-metadata')
+
+
 def git_install(projects_yaml):
     """Perform setup, and install git repos specified in yaml parameter."""
     if git_install_requested():

metadata.yaml

@@ -14,7 +14,7 @@ description: |
   .
   This charm provides the OpenStack Neutron OpenvSwitch agent, managing
   L2 connectivity on nova-compute services.
-categories:
+tags:
   - openstack
 provides:
   neutron-plugin:

templates/icehouse/dhcp_agent.ini

@@ -0,0 +1,14 @@
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+###############################################################################
+[DEFAULT]
+state_path = /var/lib/neutron
+interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
+dhcp_driver = neutron.agent.linux.dhcp.Dnsmasq
+root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
+
+enable_metadata_network = True
+enable_isolated_metadata = True
+
+ovs_use_veth = True

unit_tests/test_neutron_ovs_context.py

@@ -250,11 +250,11 @@ class L3AgentContextTest(CharmTestCase):
         self.assertEquals(context.L3AgentContext()(), {'agent_mode': 'legacy'})
 
 
-class DVRSharedSecretContext(CharmTestCase):
+class SharedSecretContext(CharmTestCase):
 
     def setUp(self):
-        super(DVRSharedSecretContext, self).setUp(context,
+        super(SharedSecretContext, self).setUp(context,
-                                                  TO_PATCH)
+                                               TO_PATCH)
         self.config.side_effect = self.test_config.get
 
     @patch('os.path')
@@ -286,7 +286,7 @@ class DVRSharedSecretContext(CharmTestCase):
         _NeutronAPIContext.side_effect = fake_context({'enable_dvr': True})
         _shared_secret.return_value = 'secret_thing'
         self.resolve_address.return_value = '10.0.0.10'
-        self.assertEquals(context.DVRSharedSecretContext()(),
+        self.assertEquals(context.SharedSecretContext()(),
                           {'shared_secret': 'secret_thing',
                            'local_ip': '10.0.0.10'})
 
@@ -297,4 +297,4 @@ class DVRSharedSecretContext(CharmTestCase):
         _NeutronAPIContext.side_effect = fake_context({'enable_dvr': False})
         _shared_secret.return_value = 'secret_thing'
         self.resolve_address.return_value = '10.0.0.10'
-        self.assertEquals(context.DVRSharedSecretContext()(), {})
+        self.assertEquals(context.SharedSecretContext()(), {})

unit_tests/test_neutron_ovs_hooks.py

@@ -1,4 +1,4 @@
-from mock import MagicMock, patch, call
+from mock import MagicMock, patch
 import yaml
 
 from test_utils import CharmTestCase
@@ -19,13 +19,8 @@ utils.register_configs = _reg
 utils.restart_map = _map
 
 TO_PATCH = [
-    'apt_update',
-    'apt_install',
-    'apt_purge',
     'config',
     'CONFIGS',
-    'determine_packages',
-    'determine_dvr_packages',
     'get_shared_secret',
     'git_install',
     'log',
@@ -33,6 +28,10 @@ TO_PATCH = [
     'relation_set',
     'configure_ovs',
     'use_dvr',
+    'install_packages',
+    'purge_packages',
+    'enable_nova_metadata',
+    'enable_local_dhcp',
 ]
 NEUTRON_CONF_DIR = "/etc/neutron"
 
@@ -54,19 +53,12 @@ class NeutronOVSHooksTests(CharmTestCase):
     @patch.object(hooks, 'git_install_requested')
     def test_install_hook(self, git_requested):
         git_requested.return_value = False
-        _pkgs = ['foo', 'bar']
-        self.determine_packages.return_value = [_pkgs]
         self._call_hook('install')
-        self.apt_update.assert_called_with()
+        self.install_packages.assert_called_with()
-        self.apt_install.assert_has_calls([
-            call(_pkgs, fatal=True),
-        ])
 
     @patch.object(hooks, 'git_install_requested')
     def test_install_hook_git(self, git_requested):
         git_requested.return_value = True
-        _pkgs = ['foo', 'bar']
-        self.determine_packages.return_value = _pkgs
         openstack_origin_git = {
             'repositories': [
                 {'name': 'requirements',
@@ -81,8 +73,7 @@ class NeutronOVSHooksTests(CharmTestCase):
         projects_yaml = yaml.dump(openstack_origin_git)
         self.test_config.set('openstack-origin-git', projects_yaml)
         self._call_hook('install')
-        self.apt_update.assert_called_with()
+        self.install_packages.assert_called_with()
-        self.assertTrue(self.determine_packages)
         self.git_install.assert_called_with(projects_yaml)
 
     @patch.object(hooks, 'git_install_requested')
@@ -124,13 +115,9 @@ class NeutronOVSHooksTests(CharmTestCase):
     @patch.object(hooks, 'git_install_requested')
     def test_config_changed_dvr(self, git_requested):
         git_requested.return_value = False
-        self.determine_dvr_packages.return_value = ['dvr']
         self._call_hook('config-changed')
-        self.apt_update.assert_called_with()
+        self.install_packages.assert_called_with()
         self.assertTrue(self.CONFIGS.write_all.called)
-        self.apt_install.assert_has_calls([
-            call(['dvr'], fatal=True),
-        ])
         self.configure_ovs.assert_called_with()
 
     @patch.object(hooks, 'neutron_plugin_joined')
@@ -140,9 +127,22 @@ class NeutronOVSHooksTests(CharmTestCase):
         self.configure_ovs.assert_called_with()
         self.assertTrue(self.CONFIGS.write_all.called)
         _plugin_joined.assert_called_with(relation_id='rid')
+        self.install_packages.assert_called_with()
+
+    @patch.object(hooks, 'neutron_plugin_joined')
+    def test_neutron_plugin_api_nodvr(self, _plugin_joined):
+        self.use_dvr.return_value = False
+        self.relation_ids.return_value = ['rid']
+        self._call_hook('neutron-plugin-api-relation-changed')
+        self.configure_ovs.assert_called_with()
+        self.assertTrue(self.CONFIGS.write_all.called)
+        _plugin_joined.assert_called_with(relation_id='rid')
+        self.purge_packages.assert_called_with(['neutron-l3-agent'])
 
     @patch.object(hooks, 'git_install_requested')
     def test_neutron_plugin_joined(self, git_requested):
+        self.enable_nova_metadata.return_value = True
+        self.enable_local_dhcp.return_value = True
         git_requested.return_value = False
         self.get_shared_secret.return_value = 'secret'
         self._call_hook('neutron-plugin-relation-joined')

unit_tests/test_neutron_ovs_utils.py

@@ -18,8 +18,11 @@ import charmhelpers.core.hookenv as hookenv
 TO_PATCH = [
     'add_bridge',
     'add_bridge_port',
+    'apt_install',
+    'apt_update',
     'config',
     'os_release',
+    'filter_installed_packages',
     'neutron_plugin_attribute',
     'full_restart',
     'service_restart',
@@ -75,20 +78,58 @@ class TestNeutronOVSUtils(CharmTestCase):
         # Reset cached cache
         hookenv.cache = {}
 
+    @patch.object(nutils, 'determine_packages')
+    def test_install_packages(self, _determine_packages):
+        _determine_packages.return_value = 'randompkg'
+        nutils.install_packages()
+        self.apt_update.assert_called_with()
+        self.apt_install.assert_called_with(self.filter_installed_packages())
+
     @patch.object(nutils, 'use_dvr')
     @patch.object(nutils, 'git_install_requested')
     @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release')
     @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package')
     def test_determine_packages(self, _head_pkgs, _os_rel, _git_requested,
                                 _use_dvr):
+        self.test_config.set('enable-local-dhcp-and-metadata', False)
         _git_requested.return_value = False
         _use_dvr.return_value = False
         _os_rel.return_value = 'trusty'
         _head_pkgs.return_value = head_pkg
         pkg_list = nutils.determine_packages()
-        expect = [['neutron-plugin-openvswitch-agent'], [head_pkg]]
+        expect = ['neutron-plugin-openvswitch-agent', head_pkg]
         self.assertItemsEqual(pkg_list, expect)
 
+    @patch.object(nutils, 'use_dvr')
+    @patch.object(nutils, 'git_install_requested')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package')
+    def test_determine_packages_metadata(self, _head_pkgs, _os_rel,
+                                         _git_requested, _use_dvr):
+        self.test_config.set('enable-local-dhcp-and-metadata', True)
+        _git_requested.return_value = False
+        _use_dvr.return_value = False
+        _os_rel.return_value = 'trusty'
+        _head_pkgs.return_value = head_pkg
+        pkg_list = nutils.determine_packages()
+        expect = ['neutron-plugin-openvswitch-agent', head_pkg,
+                  'neutron-metadata-agent', 'neutron-dhcp-agent']
+        self.assertItemsEqual(pkg_list, expect)
+
+    @patch.object(nutils, 'use_dvr')
+    @patch.object(nutils, 'git_install_requested')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package')
+    def test_determine_packages_git(self, _head_pkgs, _os_rel,
+                                    _git_requested, _use_dvr):
+        self.test_config.set('enable-local-dhcp-and-metadata', False)
+        _git_requested.return_value = True
+        _use_dvr.return_value = True
+        _os_rel.return_value = 'trusty'
+        _head_pkgs.return_value = head_pkg
+        pkg_list = nutils.determine_packages()
+        self.assertFalse('neutron-l3-agent' in pkg_list)
+
     @patch.object(nutils, 'use_dvr')
     def test_register_configs(self, _use_dvr):
         class _mock_OSConfigRenderer():
@@ -128,6 +169,19 @@ class TestNeutronOVSUtils(CharmTestCase):
         [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
         self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs)
 
+    @patch.object(nutils, 'enable_local_dhcp')
+    @patch.object(nutils, 'use_dvr')
+    def test_resource_map_dhcp(self, _use_dvr, _enable_local_dhcp):
+        _enable_local_dhcp.return_value = True
+        _use_dvr.return_value = False
+        _map = nutils.resource_map()
+        svcs = ['neutron-plugin-openvswitch-agent', 'neutron-metadata-agent',
+                'neutron-dhcp-agent']
+        confs = [nutils.NEUTRON_CONF, nutils.NEUTRON_METADATA_AGENT_CONF,
+                 nutils.NEUTRON_DHCP_AGENT_CONF]
+        [self.assertIn(q_conf, _map.keys()) for q_conf in confs]
+        self.assertEqual(_map[nutils.NEUTRON_CONF]['services'], svcs)
+
     @patch.object(nutils, 'use_dvr')
     def test_restart_map(self, _use_dvr):
         _use_dvr.return_value = False
@@ -213,7 +267,7 @@ class TestNeutronOVSUtils(CharmTestCase):
         ])
         self.add_bridge_port.assert_called_with('br-ex', 'eth0')
 
-    @patch.object(neutron_ovs_context, 'DVRSharedSecretContext')
+    @patch.object(neutron_ovs_context, 'SharedSecretContext')
     def test_get_shared_secret(self, _dvr_secret_ctxt):
         _dvr_secret_ctxt.return_value = \
             DummyContext(return_value={'shared_secret': 'supersecret'})