Merge "Create utility to clean-up netns."

bin/quantum-netns-cleanup

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2012 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from quantum.agent.netns_cleanup_util import main
+main()

quantum/agent/linux/interface.py

@@ -154,6 +154,10 @@ class OVSInterfaceDriver(LinuxInterfaceDriver):
         bridge = ovs_lib.OVSBridge(bridge, self.conf.root_helper)
         bridge.delete_port(device_name)
 
+        if namespace:
+            ip = ip_lib.IPWrapper(self.conf.root_helper, namespace)
+            ip.garbage_collect_namespace()
+
 
 class BridgeInterfaceDriver(LinuxInterfaceDriver):
     """Driver for creating bridge interfaces."""
@@ -196,6 +200,10 @@ class BridgeInterfaceDriver(LinuxInterfaceDriver):
             LOG.error(_("Failed unplugging interface '%s'") %
                       device_name)
 
+        if namespace:
+            ip = ip_lib.IPWrapper(self.conf.root_helper, namespace)
+            ip.garbage_collect_namespace()
+
 
 class RyuInterfaceDriver(OVSInterfaceDriver):
     """Driver for creating a Ryu OVS interface."""

quantum/agent/linux/ip_lib.py

@@ -18,6 +18,9 @@ from quantum.agent.linux import utils
 from quantum.common import exceptions
 
 
+LOOPBACK_DEVNAME = 'lo'
+
+
 class SubProcessBase(object):
     def __init__(self, root_helper=None, namespace=None):
         self.root_helper = root_helper
@@ -62,7 +65,7 @@ class IPWrapper(SubProcessBase):
     def device(self, name):
         return IPDevice(name, self.root_helper, self.namespace)
 
-    def get_devices(self):
+    def get_devices(self, exclude_loopback=False):
         retval = []
         output = self._execute('o', 'link', ('list',),
                                self.root_helper, self.namespace)
@@ -71,7 +74,12 @@ class IPWrapper(SubProcessBase):
                 continue
             tokens = line.split(':', 2)
             if len(tokens) >= 3:
-                retval.append(IPDevice(tokens[1].strip(),
+                name = tokens[1].strip()
+
+                if exclude_loopback and name == LOOPBACK_DEVNAME:
+                    continue
+
+                retval.append(IPDevice(name,
                                        self.root_helper,
                                        self.namespace))
         return retval
@@ -90,12 +98,23 @@ class IPWrapper(SubProcessBase):
     def ensure_namespace(self, name):
         if not self.netns.exists(name):
             ip = self.netns.add(name)
-            lo = ip.device('lo')
+            lo = ip.device(LOOPBACK_DEVNAME)
             lo.link.set_up()
         else:
             ip = IPWrapper(self.root_helper, name)
         return ip
 
+    def namespace_is_empty(self):
+        return not self.get_devices(exclude_loopback=True)
+
+    def garbage_collect_namespace(self):
+        """Conditionally destroy the namespace if it is empty."""
+        if self.namespace and self.netns.exists(self.namespace):
+            if self.namespace_is_empty():
+                self.netns.delete(self.namespace)
+                return True
+        return False
+
     def add_device_to_namespace(self, device):
         if self.namespace:
             device.link.set_netns(self.namespace)

quantum/agent/linux/ovs_lib.py

@@ -269,3 +269,12 @@ class OVSBridge:
         except Exception, e:
             LOG.info("Unable to parse regex results. Exception: %s", e)
             return
+
+
+def get_bridge_for_iface(root_helper, iface):
+    args = ["ovs-vsctl", "--timeout=2", "iface-to-br", iface]
+    try:
+        return utils.execute(args, root_helper=root_helper).strip()
+    except Exception, e:
+        LOG.error(_("iface %s not found. Exception: %s"), iface, e)
+        return None

quantum/agent/netns_cleanup_util.py

@@ -0,0 +1,162 @@
+import logging
+import os
+import re
+import sys
+import traceback
+
+import eventlet
+
+from quantum.agent import dhcp_agent
+from quantum.agent import l3_agent
+from quantum.agent.linux import dhcp
+from quantum.agent.linux import ip_lib
+from quantum.agent.linux import ovs_lib
+from quantum.api.v2 import attributes
+from quantum.common import config
+from quantum.openstack.common import cfg
+from quantum.openstack.common import importutils
+
+LOG = logging.getLogger(__name__)
+NS_MANGLING_PATTERN = ('(%s|%s)' % (dhcp_agent.NS_PREFIX, l3_agent.NS_PREFIX) +
+                       attributes.UUID_PATTERN)
+
+
+class NullDelegate(object):
+    def __getattribute__(self, name):
+        def noop(*args, **kwargs):
+            pass
+        return noop
+
+
+class FakeNetwork(object):
+    def __init__(self, id):
+        self.id = id
+
+
+def setup_conf():
+    """Setup the cfg for the clean up utility.
+
+    Use separate setup_conf for the utility because there are many options
+    from the main config that do not apply during clean-up.
+    """
+
+    opts = [
+        cfg.StrOpt('root_helper', default='sudo'),
+        cfg.StrOpt('dhcp_driver',
+                   default='quantum.agent.linux.dhcp.Dnsmasq',
+                   help="The driver used to manage the DHCP server."),
+        cfg.StrOpt('state_path',
+                   default='.',
+                   help='Top-level directory for maintaining dhcp state'),
+        cfg.BoolOpt('force',
+                    default=False,
+                    help='Delete the namespace by removing all devices.'),
+    ]
+    conf = cfg.CommonConfigOpts()
+    conf.register_opts(opts)
+    conf.register_opts(dhcp.OPTS)
+    config.setup_logging(conf)
+    return conf
+
+
+def kill_dhcp(conf, namespace):
+    """Disable DHCP for a network if DHCP is still active."""
+    network_id = namespace.replace(dhcp_agent.NS_PREFIX, '')
+
+    null_delegate = NullDelegate()
+    dhcp_driver = importutils.import_object(
+        conf.dhcp_driver,
+        conf,
+        FakeNetwork(network_id),
+        conf.root_helper,
+        null_delegate)
+
+    if dhcp_driver.active:
+        dhcp_driver.disable()
+
+
+def eligible_for_deletion(conf, namespace, force=False):
+    """Determine whether a namespace is eligible for deletion.
+
+    Eligibility is determined by having only the lo device or if force
+    is passed as a parameter.
+    """
+
+    # filter out namespaces without UUID as the name
+    if not re.match(NS_MANGLING_PATTERN, namespace):
+        return False
+
+    ip = ip_lib.IPWrapper(conf.root_helper, namespace)
+    return force or ip.namespace_is_empty()
+
+
+def unplug_device(conf, device):
+    try:
+        device.link.delete()
+    except RuntimeError:
+        # Maybe the device is OVS port, so try to delete
+        bridge_name = ovs_lib.get_bridge_for_iface(conf.root_helper,
+                                                   device.name)
+        if bridge_name:
+            bridge = ovs_lib.OVSBridge(bridge_name,
+                                       conf.root_helper)
+            bridge.delete_port(device.name)
+        else:
+            LOG.debug(_('Unable to find bridge for device: %s') % device.name)
+
+
+def destroy_namespace(conf, namespace, force=False):
+    """Destroy a given namespace.
+
+    If force is True, then dhcp (if it exists) will be disabled and all
+    devices will be forcibly removed.
+    """
+
+    try:
+        ip = ip_lib.IPWrapper(conf.root_helper, namespace)
+
+        if force:
+            kill_dhcp(conf, namespace)
+            # NOTE: The dhcp driver will remove the namespace if is it empty,
+            # so a second check is required here.
+            if ip.netns.exists(namespace):
+                for device in ip.get_devices(exclude_loopback=True):
+                    unplug_device(conf, device)
+
+        ip.garbage_collect_namespace()
+    except Exception, e:
+        LOG.exception(_('Error unable to destroy namespace: %s') % namespace)
+
+
+def main():
+    """Main method for cleaning up network namespaces.
+
+    This method will make two passes checking for namespaces to delete. The
+    process will identify candidates, sleep, and call garbage collect. The
+    garbage collection will re-verify that the namespace meets the criteria for
+    deletion (ie it is empty). The period of sleep and the 2nd pass allow
+    time for the namespace state to settle, so that the check prior deletion
+    will re-confirm the namespace is empty.
+
+    The utility is designed to clean-up after the forced or unexpected
+    termination of Quantum agents.
+
+    The --force flag should only be used as part of the cleanup of a devstack
+    installation as it will blindly purge namespaces and their devices. This
+    option also kills any lingering DHCP instances.
+    """
+    eventlet.monkey_patch()
+
+    conf = setup_conf()
+    conf(sys.argv)
+
+    # Identify namespaces that are candidates for deletion.
+    candidates = [ns for ns in
+                  ip_lib.IPWrapper.get_namespaces(conf.root_helper)
+                  if eligible_for_deletion(conf, ns, conf.force)]
+
+    if candidates:
+        eventlet.sleep(2)
+
+        for namespace in candidates:
+            destroy_namespace(conf, namespace, conf.force)

quantum/tests/unit/openvswitch/test_ovs_lib.py

@@ -15,10 +15,10 @@
 #    under the License.
 # @author: Dan Wendlandt, Nicira, Inc.
 
-import unittest
 import uuid
 
 import mox
+import unittest2 as unittest
 
 from quantum.agent.linux import ovs_lib, utils
 
@@ -292,3 +292,25 @@ class OVS_Lib_Test(unittest.TestCase):
         self.assertEqual(vif_id, '5c1321a7-c73f-4a77-95e6-9f86402e5c8f')
         self.assertEqual(port_name, 'dhc5c1321a7-c7')
         self.assertEqual(ofport, 2)
+
+    def test_iface_to_br(self):
+        iface = 'tap0'
+        br = 'br-int'
+        root_helper = 'sudo'
+        utils.execute(["ovs-vsctl", self.TO, "iface-to-br", iface],
+                      root_helper=root_helper).AndReturn('br-int')
+
+        self.mox.ReplayAll()
+        self.assertEqual(ovs_lib.get_bridge_for_iface(root_helper, iface), br)
+        self.mox.VerifyAll()
+
+    def test_iface_to_br(self):
+        iface = 'tap0'
+        br = 'br-int'
+        root_helper = 'sudo'
+        utils.execute(["ovs-vsctl", self.TO, "iface-to-br", iface],
+                      root_helper=root_helper).AndRaise(Exception)
+
+        self.mox.ReplayAll()
+        self.assertIsNone(ovs_lib.get_bridge_for_iface(root_helper, iface))
+        self.mox.VerifyAll()

quantum/tests/unit/test_agent_netns_cleanup.py

@@ -0,0 +1,230 @@
+import mock
+import unittest2 as unittest
+
+from quantum.agent import netns_cleanup_util as util
+
+
+class TestNetnsCleanup(unittest.TestCase):
+    def test_setup_conf(self):
+        conf = util.setup_conf()
+        self.assertFalse(conf.force)
+
+    def test_kill_dhcp(self, dhcp_active=True):
+        conf = mock.Mock()
+        conf.root_helper = 'sudo',
+        conf.dhcp_driver = 'driver'
+
+        method_to_patch = 'quantum.openstack.common.importutils.import_object'
+
+        with mock.patch(method_to_patch) as import_object:
+            driver = mock.Mock()
+            driver.active = dhcp_active
+            import_object.return_value = driver
+
+            util.kill_dhcp(conf, 'ns')
+
+            import_object.called_once_with('driver', conf, mock.ANY, 'sudo',
+                                           mock.ANY)
+
+            if dhcp_active:
+                driver.assert_has_calls([mock.call.disable()])
+            else:
+                self.assertFalse(driver.called)
+
+    def test_kill_dhcp_no_active(self):
+        self.test_kill_dhcp(False)
+
+    def test_eligible_for_deletion_ns_not_uuid(self):
+        ns = 'not_a_uuid'
+        self.assertFalse(util.eligible_for_deletion(mock.Mock(), ns))
+
+    def _test_eligible_for_deletion_helper(self, prefix, force, is_empty,
+                                           expected):
+        ns = prefix + '6e322ac7-ab50-4f53-9cdc-d1d3c1164b6d'
+        conf = mock.Mock()
+        conf.root_helper = 'sudo'
+
+        with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
+            ip_wrap.return_value.namespace_is_empty.return_value = is_empty
+            self.assertEqual(util.eligible_for_deletion(conf, ns, force),
+                             expected)
+
+            expected_calls = [mock.call('sudo', ns)]
+            if not force:
+                expected_calls.append(mock.call().namespace_is_empty())
+            ip_wrap.assert_has_calls(expected_calls)
+
+    def test_eligible_for_deletion_empty(self):
+        self._test_eligible_for_deletion_helper('qrouter-', False, True, True)
+
+    def test_eligible_for_deletion_not_empty(self):
+        self._test_eligible_for_deletion_helper('qdhcp-', False, False, False)
+
+    def test_eligible_for_deletion_not_empty_forced(self):
+        self._test_eligible_for_deletion_helper('qdhcp-', True, False, True)
+
+    def test_unplug_device_regular_device(self):
+        conf = mock.Mock()
+        device = mock.Mock()
+
+        util.unplug_device(conf, device)
+        device.assert_has_calls([mock.call.link.delete()])
+
+    def test_unplug_device_ovs_port(self):
+        conf = mock.Mock()
+        conf.ovs_integration_bridge = 'br-int'
+        conf.root_helper = 'sudo'
+
+        device = mock.Mock()
+        device.name = 'tap1'
+        device.link.delete.side_effect = RuntimeError
+
+        with mock.patch('quantum.agent.linux.ovs_lib.OVSBridge') as ovs_br_cls:
+            br_patch = mock.patch(
+                'quantum.agent.linux.ovs_lib.get_bridge_for_iface')
+            with br_patch as mock_get_bridge_for_iface:
+                mock_get_bridge_for_iface.return_value = 'br-int'
+                ovs_bridge = mock.Mock()
+                ovs_br_cls.return_value = ovs_bridge
+
+                util.unplug_device(conf, device)
+
+                mock_get_bridge_for_iface.assert_called_once_with(
+                    conf.root_helper, 'tap1')
+                ovs_br_cls.called_once_with('br-int', 'sudo')
+                ovs_bridge.assert_has_calls(
+                    [mock.call.delete_port(device.name)])
+
+    def test_unplug_device_cannot_determine_bridge_port(self):
+        conf = mock.Mock()
+        conf.ovs_integration_bridge = 'br-int'
+        conf.root_helper = 'sudo'
+
+        device = mock.Mock()
+        device.name = 'tap1'
+        device.link.delete.side_effect = RuntimeError
+
+        with mock.patch('quantum.agent.linux.ovs_lib.OVSBridge') as ovs_br_cls:
+            br_patch = mock.patch(
+                'quantum.agent.linux.ovs_lib.get_bridge_for_iface')
+            with br_patch as mock_get_bridge_for_iface:
+                with mock.patch.object(util.LOG, 'debug') as debug:
+                    mock_get_bridge_for_iface.return_value = None
+                    ovs_bridge = mock.Mock()
+                    ovs_br_cls.return_value = ovs_bridge
+
+                    util.unplug_device(conf, device)
+
+                    mock_get_bridge_for_iface.assert_called_once_with(
+                        conf.root_helper, 'tap1')
+                    self.assertEquals(ovs_br_cls.mock_calls, [])
+                    self.assertTrue(debug.called)
+
+    def _test_destroy_namespace_helper(self, force, num_devices):
+        ns = 'qrouter-6e322ac7-ab50-4f53-9cdc-d1d3c1164b6d'
+        conf = mock.Mock()
+        conf.root_helper = 'sudo'
+
+        lo_device = mock.Mock()
+        lo_device.name = 'lo'
+
+        devices = [lo_device]
+
+        while num_devices:
+            dev = mock.Mock()
+            dev.name = 'tap%d' % num_devices
+            devices.append(dev)
+            num_devices -= 1
+
+        with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
+            ip_wrap.return_value.get_devices.return_value = devices
+            ip_wrap.return_value.netns.exists.return_value = True
+
+            with mock.patch.object(util, 'unplug_device') as unplug:
+
+                with mock.patch.object(util, 'kill_dhcp') as kill_dhcp:
+                    util.destroy_namespace(conf, ns, force)
+                    expected = [mock.call('sudo', ns)]
+
+                    if force:
+                        expected.extend([
+                            mock.call().netns.exists(ns),
+                            mock.call().get_devices(exclude_loopback=True)])
+                        self.assertTrue(kill_dhcp.called)
+                        unplug.assert_has_calls(
+                            [mock.call(conf, d) for d in
+                             devices[1:]])
+
+                    expected.append(mock.call().garbage_collect_namespace())
+                    ip_wrap.assert_has_calls(expected)
+
+    def test_destory_namespace_empty(self):
+        self._test_destroy_namespace_helper(False, 0)
+
+    def test_destory_namespace_not_empty(self):
+        self._test_destroy_namespace_helper(False, 1)
+
+    def test_destory_namespace_not_empty_forced(self):
+        self._test_destroy_namespace_helper(True, 2)
+
+    def test_main(self):
+        namespaces = ['ns1', 'ns2']
+        with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
+            ip_wrap.get_namespaces.return_value = namespaces
+
+            with mock.patch('eventlet.sleep') as eventlet_sleep:
+                conf = mock.Mock()
+                conf.root_helper = 'sudo'
+                conf.force = False
+                methods_to_mock = dict(
+                    eligible_for_deletion=mock.DEFAULT,
+                    destroy_namespace=mock.DEFAULT,
+                    setup_conf=mock.DEFAULT)
+
+                with mock.patch.multiple(util, **methods_to_mock) as mocks:
+                    mocks['eligible_for_deletion'].return_value = True
+                    mocks['setup_conf'].return_value = conf
+                    util.main()
+
+                    mocks['eligible_for_deletion'].assert_has_calls(
+                        [mock.call(conf, 'ns1', False),
+                         mock.call(conf, 'ns2', False)])
+
+                    mocks['destroy_namespace'].assert_has_calls(
+                        [mock.call(conf, 'ns1', False),
+                         mock.call(conf, 'ns2', False)])
+
+                    ip_wrap.assert_has_calls(
+                        [mock.call.get_namespaces('sudo')])
+
+                    eventlet_sleep.assert_called_once_with(2)
+
+    def test_main_no_candidates(self):
+        namespaces = ['ns1', 'ns2']
+        with mock.patch('quantum.agent.linux.ip_lib.IPWrapper') as ip_wrap:
+            ip_wrap.get_namespaces.return_value = namespaces
+
+            with mock.patch('eventlet.sleep') as eventlet_sleep:
+                conf = mock.Mock()
+                conf.root_helper = 'sudo'
+                conf.force = False
+                methods_to_mock = dict(
+                    eligible_for_deletion=mock.DEFAULT,
+                    destroy_namespace=mock.DEFAULT,
+                    setup_conf=mock.DEFAULT)
+
+                with mock.patch.multiple(util, **methods_to_mock) as mocks:
+                    mocks['eligible_for_deletion'].return_value = False
+                    mocks['setup_conf'].return_value = conf
+                    util.main()
+
+                    ip_wrap.assert_has_calls(
+                        [mock.call.get_namespaces('sudo')])
+
+                    mocks['eligible_for_deletion'].assert_has_calls(
+                        [mock.call(conf, 'ns1', False),
+                         mock.call(conf, 'ns2', False)])
+
+                    self.assertFalse(mocks['destroy_namespace'].called)
+
+                    self.assertFalse(eventlet_sleep.called)

quantum/tests/unit/test_linux_ip_lib.py

@@ -206,6 +206,72 @@ class TestIpWrapper(unittest.TestCase):
             self.assertFalse(self.execute.called)
             self.assertEqual(ns.namespace, 'ns')
 
+    def test_namespace_is_empty_no_devices(self):
+        ip = ip_lib.IPWrapper('sudo', 'ns')
+        with mock.patch.object(ip, 'get_devices') as get_devices:
+            get_devices.return_value = []
+
+            self.assertTrue(ip.namespace_is_empty())
+            get_devices.assert_called_once_with(exclude_loopback=True)
+
+    def test_namespace_is_empty(self):
+        ip = ip_lib.IPWrapper('sudo', 'ns')
+        with mock.patch.object(ip, 'get_devices') as get_devices:
+            get_devices.return_value = [mock.Mock()]
+
+            self.assertFalse(ip.namespace_is_empty())
+            get_devices.assert_called_once_with(exclude_loopback=True)
+
+    def test_garbage_collect_namespace_does_not_exist(self):
+        with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
+            ip_ns_cmd_cls.return_value.exists.return_value = False
+            ip = ip_lib.IPWrapper('sudo', 'ns')
+            with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
+
+                self.assertFalse(ip.garbage_collect_namespace())
+                ip_ns_cmd_cls.assert_has_calls([mock.call().exists('ns')])
+                self.assertNotIn(mock.call().delete('ns'),
+                                 ip_ns_cmd_cls.return_value.mock_calls)
+                self.assertEqual(mock_is_empty.mock_calls, [])
+
+    def test_garbage_collect_namespace_existing_empty_ns(self):
+        with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
+            ip_ns_cmd_cls.return_value.exists.return_value = True
+
+            ip = ip_lib.IPWrapper('sudo', 'ns')
+
+            with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
+                mock_is_empty.return_value = True
+                self.assertTrue(ip.garbage_collect_namespace())
+
+                mock_is_empty.assert_called_once_with()
+                expected = [mock.call().exists('ns'),
+                            mock.call().delete('ns')]
+                ip_ns_cmd_cls.assert_has_calls(expected)
+
+    def test_garbage_collect_namespace_existing_not_empty(self):
+        lo_device = mock.Mock()
+        lo_device.name = 'lo'
+        tap_device = mock.Mock()
+        tap_device.name = 'tap1'
+
+        with mock.patch.object(ip_lib, 'IpNetnsCommand') as ip_ns_cmd_cls:
+            ip_ns_cmd_cls.return_value.exists.return_value = True
+
+            ip = ip_lib.IPWrapper('sudo', 'ns')
+
+            with mock.patch.object(ip, 'namespace_is_empty') as mock_is_empty:
+                mock_is_empty.return_value = False
+
+                self.assertFalse(ip.garbage_collect_namespace())
+
+                mock_is_empty.assert_called_once_with()
+                expected = [mock.call(ip),
+                            mock.call().exists('ns')]
+                self.assertEqual(ip_ns_cmd_cls.mock_calls, expected)
+                self.assertNotIn(mock.call().delete('ns'),
+                                 ip_ns_cmd_cls.mock_calls)
+
     def test_add_device_to_namespace(self):
         dev = mock.Mock()
         ip_lib.IPWrapper('sudo', 'ns').add_device_to_namespace(dev)

setup.py

@@ -102,6 +102,7 @@ setuptools.setup(
             'quantum-dhcp-agent = quantum.agent.dhcp_agent:main',
             'quantum-dhcp-agent-dnsmasq-lease-update ='
             'quantum.agent.linux.dhcp:Dnsmasq.lease_update',
+            'quantum-netns-cleanup = quantum.agent.netns_cleanup_util:main',
             'quantum-l3-agent = quantum.agent.l3_nat_agent:main',
             'quantum-linuxbridge-agent ='
             'quantum.plugins.linuxbridge.agent.linuxbridge_quantum_agent:main',