From c3c8d1f77afc16a9117893feb3ab66efb590f6a4 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 29 Aug 2013 20:50:55 +0200 Subject: [PATCH] Avoid race with udev during ovs agent startup After taking down the veth link between the physical bridge and the integration bridge call udevadm settle to wait for any udev events to be completely processed by the operating system before recreating the veth pair. Some distributions (e.g. openSUSE) have udev rules installed by default that call e.g. ifdown during the remove event. If that is processed after the ovs agent already brought up the veth pair again the veth pair's link will be down after the agent completed startup and networking will be broken for all VM instances. Change-Id: I95520ea96a9804c5261a0c994bbca137535cc37c Closes-Bug: #1218556 --- .../openvswitch/agent/ovs_neutron_agent.py | 5 +++++ .../unit/openvswitch/test_ovs_neutron_agent.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py index eefe384367..b3fad63a7d 100644 --- a/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py +++ b/neutron/plugins/openvswitch/agent/ovs_neutron_agent.py @@ -32,6 +32,7 @@ from oslo.config import cfg from neutron.agent import l2population_rpc from neutron.agent.linux import ip_lib from neutron.agent.linux import ovs_lib +from neutron.agent.linux import utils from neutron.agent import rpc as agent_rpc from neutron.agent import securitygroups_rpc as sg_rpc from neutron.common import config as logging_config @@ -778,6 +779,10 @@ class OVSNeutronAgent(sg_rpc.SecurityGroupAgentRpcCallbackMixin, br.delete_port(phys_veth_name) if ip_lib.device_exists(int_veth_name, self.root_helper): ip_lib.IPDevice(int_veth_name, self.root_helper).link.delete() + # Give udev a chance to process its rules here, to avoid + # race conditions between commands launched by udev rules + # and the subsequent call to ip_wrapper.add_veth + utils.execute(['/sbin/udevadm', 'settle', '--timeout=10']) int_veth, phys_veth = ip_wrapper.add_veth(int_veth_name, phys_veth_name) self.int_ofports[physical_network] = self.int_br.add_port(int_veth) diff --git a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py index e74fc31495..8b415b2e03 100644 --- a/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py +++ b/neutron/tests/unit/openvswitch/test_ovs_neutron_agent.py @@ -23,6 +23,7 @@ import testtools from neutron.agent.linux import ip_lib from neutron.agent.linux import ovs_lib +from neutron.agent.linux import utils from neutron.common import constants as n_const from neutron.openstack.common.rpc import common as rpc_common from neutron.plugins.openvswitch.agent import ovs_neutron_agent @@ -327,6 +328,7 @@ class TestOvsNeutronAgent(base.BaseTestCase): with contextlib.nested( mock.patch.object(ip_lib, "device_exists"), mock.patch.object(sys, "exit"), + mock.patch.object(utils, "execute"), mock.patch.object(ovs_lib.OVSBridge, "remove_all_flows"), mock.patch.object(ovs_lib.OVSBridge, "add_flow"), mock.patch.object(ovs_lib.OVSBridge, "add_port"), @@ -337,15 +339,26 @@ class TestOvsNeutronAgent(base.BaseTestCase): mock.patch.object(ip_lib.IpLinkCommand, "delete"), mock.patch.object(ip_lib.IpLinkCommand, "set_up"), mock.patch.object(ip_lib.IpLinkCommand, "set_mtu") - ) as (devex_fn, sysexit_fn, remflows_fn, ovs_addfl_fn, + ) as (devex_fn, sysexit_fn, utilsexec_fn, remflows_fn, ovs_addfl_fn, ovs_addport_fn, ovs_delport_fn, br_addport_fn, br_delport_fn, addveth_fn, linkdel_fn, linkset_fn, linkmtu_fn): devex_fn.return_value = True + parent = mock.MagicMock() + parent.attach_mock(utilsexec_fn, 'utils_execute') + parent.attach_mock(linkdel_fn, 'link_delete') + parent.attach_mock(addveth_fn, 'add_veth') addveth_fn.return_value = (ip_lib.IPDevice("int-br-eth1"), ip_lib.IPDevice("phy-br-eth1")) ovs_addport_fn.return_value = "int_ofport" br_addport_fn.return_value = "phys_veth" self.agent.setup_physical_bridges({"physnet1": "br-eth"}) + expected_calls = [mock.call.link_delete(), + mock.call.utils_execute(['/sbin/udevadm', + 'settle', + '--timeout=10']), + mock.call.add_veth('int-br-eth', + 'phy-br-eth')] + parent.assert_has_calls(expected_calls, any_order=False) self.assertEqual(self.agent.int_ofports["physnet1"], "phys_veth") self.assertEqual(self.agent.phys_ofports["physnet1"],