From 8883e3f305a00d0cee12a303009f657bed240c74 Mon Sep 17 00:00:00 2001 From: Hamdy Khader Date: Mon, 13 May 2019 13:47:26 +0300 Subject: [PATCH] OVS DPDK port representors support Adds support for OVS DPDK port representors[1], a direct port on a netdev datapath is considered a DPDK representor port. Using VIFHostDevice object with netdev in its profile means the port is a DPDK representor port. [1] http://docs.openvswitch.org/en/latest/topics/dpdk/phy/#representors Closes-Bug: #1829734 Change-Id: I78e7dadfa44ac7e0ba6c9f31b3070011e783589f --- vif_plug_ovs/constants.py | 2 + vif_plug_ovs/linux_net.py | 16 ++++ vif_plug_ovs/ovs.py | 53 +++++++++----- vif_plug_ovs/ovsdb/ovsdb_lib.py | 29 +++++++- .../tests/unit/ovsdb/test_ovsdb_lib.py | 32 ++++++++ vif_plug_ovs/tests/unit/test_plugin.py | 73 +++++++++++++++++++ 6 files changed, 186 insertions(+), 19 deletions(-) diff --git a/vif_plug_ovs/constants.py b/vif_plug_ovs/constants.py index 8bb6f3db..1eece80c 100644 --- a/vif_plug_ovs/constants.py +++ b/vif_plug_ovs/constants.py @@ -22,5 +22,7 @@ OVS_DATAPATH_NETDEV = 'netdev' PLATFORM_LINUX = 'linux2' PLATFORM_WIN32 = 'win32' +OVS_DPDK_INTERFACE_TYPE = 'dpdk' + # Neutron dead VLAN. DEAD_VLAN = 4095 diff --git a/vif_plug_ovs/linux_net.py b/vif_plug_ovs/linux_net.py index e3e26729..e83fcdd5 100644 --- a/vif_plug_ovs/linux_net.py +++ b/vif_plug_ovs/linux_net.py @@ -47,6 +47,7 @@ PF_RE = re.compile(r"pf(\d+)", re.IGNORECASE) PF_FUNC_RE = re.compile(r"\.(\d+)", 0) _SRIOV_TOTALVFS = "sriov_totalvfs" +NIC_NAME_LEN = 14 def _update_device_mtu(dev, mtu): @@ -370,3 +371,18 @@ def get_vf_num_by_pci_address(pci_addr): if vf_num is None: raise exception.PciDeviceNotFoundById(id=pci_addr) return vf_num + + +def get_dpdk_representor_port_name(port_id): + devname = "vfr" + port_id + return devname[:NIC_NAME_LEN] + + +def get_pf_pci_from_vf(vf_pci): + """Get physical function PCI address of a VF + + :param vf_pci: the PCI address of the VF + :return: the PCI address of the PF + """ + physfn_path = os.readlink("/sys/bus/pci/devices/%s/physfn" % vf_pci) + return os.path.basename(physfn_path) diff --git a/vif_plug_ovs/ovs.py b/vif_plug_ovs/ovs.py index f9abe3fa..9516f79f 100644 --- a/vif_plug_ovs/ovs.py +++ b/vif_plug_ovs/ovs.py @@ -252,16 +252,28 @@ class OvsPlugin(plugin.PluginBase): vif.port_profile.create_port): self._create_vif_port(vif, vif.vif_name, instance_info) - def _plug_vf_passthrough(self, vif, instance_info): - self.ovsdb.ensure_ovs_bridge( - vif.network.bridge, constants.OVS_DATAPATH_SYSTEM) + def _plug_vf(self, vif, instance_info): + datapath = self._get_vif_datapath_type(vif) + self.ovsdb.ensure_ovs_bridge(vif.network.bridge, datapath) pci_slot = vif.dev_address - pf_ifname = linux_net.get_ifname_by_pci_address( - pci_slot, pf_interface=True, switchdev=True) vf_num = linux_net.get_vf_num_by_pci_address(pci_slot) - representor = linux_net.get_representor_port(pf_ifname, vf_num) - linux_net.set_interface_state(representor, 'up') - self._create_vif_port(vif, representor, instance_info) + args = [] + kwargs = {} + if datapath == constants.OVS_DATAPATH_SYSTEM: + pf_ifname = linux_net.get_ifname_by_pci_address( + pci_slot, pf_interface=True, switchdev=True) + representor = linux_net.get_representor_port(pf_ifname, vf_num) + linux_net.set_interface_state(representor, 'up') + args = [vif, representor, instance_info] + else: + representor = linux_net.get_dpdk_representor_port_name( + vif.id) + pf_pci = linux_net.get_pf_pci_from_vf(pci_slot) + args = [vif, representor, instance_info] + kwargs = {'interface_type': constants.OVS_DPDK_INTERFACE_TYPE, + 'pf_pci': pf_pci, + 'vf_num': vf_num} + self._create_vif_port(*args, **kwargs) def plug(self, vif, instance_info): if not hasattr(vif, "port_profile"): @@ -284,7 +296,7 @@ class OvsPlugin(plugin.PluginBase): elif isinstance(vif, objects.vif.VIFVHostUser): self._plug_vhostuser(vif, instance_info) elif isinstance(vif, objects.vif.VIFHostDevice): - self._plug_vf_passthrough(vif, instance_info) + self._plug_vf(vif, instance_info) def _unplug_vhostuser(self, vif, instance_info): self.ovsdb.delete_ovs_vif_port(vif.network.bridge, @@ -317,19 +329,26 @@ class OvsPlugin(plugin.PluginBase): # so this is not removed. self.ovsdb.delete_ovs_vif_port(vif.network.bridge, vif.vif_name) - def _unplug_vf_passthrough(self, vif, instance_info): + def _unplug_vf(self, vif): """Remove port from OVS.""" - pci_slot = vif.dev_address - pf_ifname = linux_net.get_ifname_by_pci_address(pci_slot, - pf_interface=True, switchdev=True) - vf_num = linux_net.get_vf_num_by_pci_address(pci_slot) - representor = linux_net.get_representor_port(pf_ifname, vf_num) + datapath = self._get_vif_datapath_type(vif) + if datapath == constants.OVS_DATAPATH_SYSTEM: + pci_slot = vif.dev_address + pf_ifname = linux_net.get_ifname_by_pci_address( + pci_slot, pf_interface=True, switchdev=True) + vf_num = linux_net.get_vf_num_by_pci_address(pci_slot) + representor = linux_net.get_representor_port(pf_ifname, vf_num) + else: + representor = linux_net.get_dpdk_representor_port_name( + vif.id) + # The representor interface can't be deleted because it bind the # SR-IOV VF, therefore we just need to remove it from the ovs bridge # and set the status to down self.ovsdb.delete_ovs_vif_port( vif.network.bridge, representor, delete_netdev=False) - linux_net.set_interface_state(representor, 'down') + if datapath == constants.OVS_DATAPATH_SYSTEM: + linux_net.set_interface_state(representor, 'down') def unplug(self, vif, instance_info): if not hasattr(vif, "port_profile"): @@ -352,4 +371,4 @@ class OvsPlugin(plugin.PluginBase): elif isinstance(vif, objects.vif.VIFVHostUser): self._unplug_vhostuser(vif, instance_info) elif isinstance(vif, objects.vif.VIFHostDevice): - self._unplug_vf_passthrough(vif, instance_info) + self._unplug_vf(vif) diff --git a/vif_plug_ovs/ovsdb/ovsdb_lib.py b/vif_plug_ovs/ovsdb/ovsdb_lib.py index 3ef05517..2d5cc158 100644 --- a/vif_plug_ovs/ovsdb/ovsdb_lib.py +++ b/vif_plug_ovs/ovsdb/ovsdb_lib.py @@ -62,7 +62,27 @@ class BaseOVS(object): def create_ovs_vif_port(self, bridge, dev, iface_id, mac, instance_id, mtu=None, interface_type=None, - vhost_server_path=None, tag=None): + vhost_server_path=None, tag=None, + pf_pci=None, vf_num=None): + """Create OVS port + + :param bridge: bridge name to create the port on. + :param dev: port name. + :param iface_id: port ID. + :param mac: port MAC. + :param instance_id: VM ID on which the port is attached to. + :param mtu: port MTU. + :param interface_type: OVS interface type. + :param vhost_server_path: path to socket file of vhost server. + :param tag: OVS interface tag. + :param pf_pci: PCI address of PF for dpdk representor port. + :param vf_num: VF number of PF for dpdk representor port. + + .. note:: create DPDK representor port by setting all three values: + `interface_type`, `pf_pci` and `vf_num`. if interface type is + not `OVS_DPDK_INTERFACE_TYPE` then `pf_pci` and `vf_num` values + are ignored. + """ external_ids = {'iface-id': iface_id, 'iface-status': 'active', 'attached-mac': mac, @@ -75,7 +95,12 @@ class BaseOVS(object): {'vhost-server-path': vhost_server_path})) if tag: col_values.append(('tag', tag)) - + if (interface_type == constants.OVS_DPDK_INTERFACE_TYPE and + pf_pci and vf_num): + devargs_string = "{PF_PCI},representor=[{VF_NUM}]".format( + PF_PCI=pf_pci, VF_NUM=vf_num) + col_values.append(('options', + {'dpdk-devargs': devargs_string})) with self.ovsdb.transaction() as txn: txn.add(self.ovsdb.add_port(bridge, dev)) txn.add(self.ovsdb.db_set('Interface', dev, *col_values)) diff --git a/vif_plug_ovs/tests/unit/ovsdb/test_ovsdb_lib.py b/vif_plug_ovs/tests/unit/ovsdb/test_ovsdb_lib.py index 7ea2a8c3..19e12dc6 100644 --- a/vif_plug_ovs/tests/unit/ovsdb/test_ovsdb_lib.py +++ b/vif_plug_ovs/tests/unit/ovsdb/test_ovsdb_lib.py @@ -115,6 +115,38 @@ class BaseOVSTest(testtools.TestCase): mock_update_device_mtu.assert_has_calls( [mock.call(device, mtu, interface_type=interface_type)]) + def test_create_ovs_vif_port_type_dpdk(self): + iface_id = 'iface_id' + mac = 'ca:fe:ca:fe:ca:fe' + instance_id = uuidutils.generate_uuid() + interface_type = constants.OVS_DPDK_INTERFACE_TYPE + device = 'device' + bridge = 'bridge' + mtu = 1500 + pf_pci = '0000:02:00.1' + vf_num = '0' + external_ids = {'iface-id': iface_id, + 'iface-status': 'active', + 'attached-mac': mac, + 'vm-uuid': instance_id} + values = [('external_ids', external_ids), + ('type', interface_type), + ('options', {'dpdk-devargs': + '0000:02:00.1,representor=[0]'})] + with mock.patch.object(self.br, 'update_device_mtu', + return_value=True) as mock_update_device_mtu, \ + mock.patch.object(self.br, '_ovs_supports_mtu_requests', + return_value=True): + self.br.create_ovs_vif_port(bridge, device, iface_id, mac, + instance_id, mtu=mtu, + interface_type=interface_type, + pf_pci=pf_pci, vf_num=vf_num) + self.mock_add_port.assert_has_calls([mock.call(bridge, device)]) + self.mock_db_set.assert_has_calls( + [mock.call('Interface', device, *values)]) + mock_update_device_mtu.assert_has_calls( + [mock.call(device, mtu, interface_type=interface_type)]) + def test_update_ovs_vif_port(self): with mock.patch.object(self.br, 'update_device_mtu') as \ mock_update_device_mtu: diff --git a/vif_plug_ovs/tests/unit/test_plugin.py b/vif_plug_ovs/tests/unit/test_plugin.py index 4921b31e..64f0d73a 100644 --- a/vif_plug_ovs/tests/unit/test_plugin.py +++ b/vif_plug_ovs/tests/unit/test_plugin.py @@ -61,6 +61,10 @@ class PluginTest(testtools.TestCase): interface_id='e65867e0-9340-4a7f-a256-09af6eb7a3aa', datapath_type='netdev') + self.profile_ovs_system = objects.vif.VIFPortProfileOpenVSwitch( + interface_id='e65867e0-9340-4a7f-a256-09af6eb7a3aa', + datapath_type='system') + self.profile_ovs_smart_nic = objects.vif.VIFPortProfileOpenVSwitch( interface_id='e65867e0-9340-4a7f-a256-09af6eb7a3aa', create_port=True) @@ -114,6 +118,14 @@ class PluginTest(testtools.TestCase): dev_type=fields.VIFHostDeviceDevType.ETHERNET, dev_address='0002:24:12.3', bridge_name='br-int', + port_profile=self.profile_ovs_system) + + self.vif_ovs_vf_dpdk = objects.vif.VIFHostDevice( + id='b679325f-ca89-4ee0-a8be-6db1409b69ea', + address='ca:fe:de:ad:be:ef', + network=self.network_ovs, + dev_type=fields.VIFHostDeviceDevType.ETHERNET, + dev_address='0002:24:12.3', port_profile=self.profile_ovs) self.instance = objects.instance_info.InstanceInfo( @@ -470,3 +482,64 @@ class PluginTest(testtools.TestCase): plugin = ovs.OvsPlugin.load(constants.PLUGIN_NAME) plugin.unplug(self.vif_ovs_smart_nic, self.instance) delete_port.assert_called_once() + + @mock.patch.object(linux_net, 'get_dpdk_representor_port_name') + @mock.patch.object(ovsdb_lib.BaseOVS, 'ensure_ovs_bridge') + @mock.patch.object(linux_net, 'get_vf_num_by_pci_address') + @mock.patch.object(linux_net, 'get_pf_pci_from_vf') + @mock.patch.object(ovs.OvsPlugin, '_create_vif_port') + def test_plug_ovs_vf_dpdk(self, _create_vif_port, + get_pf_pci_from_vf, + get_vf_num_by_pci_address, + ensure_ovs_bridge, + get_dpdk_representor_port_name): + + pf_pci = self.vif_ovs_vf_dpdk.dev_address + devname = 'vfrb679325f-ca' + get_vf_num_by_pci_address.return_value = '2' + get_pf_pci_from_vf.return_value = pf_pci + get_dpdk_representor_port_name.return_value = devname + calls = { + 'ensure_ovs_bridge': [mock.call('br0', + constants.OVS_DATAPATH_NETDEV)], + 'get_vf_num_by_pci_address': [mock.call('0002:24:12.3')], + 'get_pf_pci_from_vf': [mock.call(pf_pci)], + 'get_dpdk_representor_port_name': [mock.call( + self.vif_ovs_vf_dpdk.id)], + '_create_vif_port': [mock.call( + self.vif_ovs_vf_dpdk, + devname, + self.instance, + interface_type='dpdk', + pf_pci=pf_pci, + vf_num='2')]} + + plugin = ovs.OvsPlugin.load(constants.PLUGIN_NAME) + plugin.plug(self.vif_ovs_vf_dpdk, self.instance) + ensure_ovs_bridge.assert_has_calls( + calls['ensure_ovs_bridge']) + get_vf_num_by_pci_address.assert_has_calls( + calls['get_vf_num_by_pci_address']) + get_pf_pci_from_vf.assert_has_calls( + calls['get_pf_pci_from_vf']) + get_dpdk_representor_port_name.assert_has_calls( + calls['get_dpdk_representor_port_name']) + _create_vif_port.assert_has_calls( + calls['_create_vif_port']) + + @mock.patch.object(linux_net, 'get_dpdk_representor_port_name') + @mock.patch.object(ovsdb_lib.BaseOVS, 'delete_ovs_vif_port') + def test_unplug_ovs_vf_dpdk(self, delete_ovs_vif_port, + get_dpdk_representor_port_name): + devname = 'vfrb679325f-ca' + get_dpdk_representor_port_name.return_value = devname + calls = { + 'get_dpdk_representor_port_name': [mock.call( + self.vif_ovs_vf_dpdk.id)], + 'delete_ovs_vif_port': [mock.call('br0', devname, + delete_netdev=False)]} + plugin = ovs.OvsPlugin.load(constants.PLUGIN_NAME) + plugin.unplug(self.vif_ovs_vf_dpdk, self.instance) + get_dpdk_representor_port_name.assert_has_calls( + calls['get_dpdk_representor_port_name']) + delete_ovs_vif_port.assert_has_calls(calls['delete_ovs_vif_port'])