diff --git a/doc/source/plugins/ovs.rst b/doc/source/plugins/ovs.rst index dc360526..a39d9e93 100644 --- a/doc/source/plugins/ovs.rst +++ b/doc/source/plugins/ovs.rst @@ -32,6 +32,24 @@ The Open vSwitch plugin provides support for the following VIF types: Refer to :ref:`vif-vhostuser` for more information. +`VIFHostDevice` + + Configuration where an SR-IOV PCI device virtual function (VF) is passed + through to a guest. The ``hw-tc-offload`` feature should be enabled on the + SR-IOV Physical Function (PF) using ``ethtool``: + + .. code-block:: shell + + ethtool -K hw-tc-offload + + This will create *VF representor* per VF. The VF representor plays the same + role as TAP devices in Para-Virtual (PV) setup. In this case the ``plug()`` + method connects the VF representor to the OpenVSwitch bridge. + + Refer to :ref:`vif-hostdevice` for more information. + + .. versionadded:: 1.5.0 + For information on the VIF type objects, refer to :doc:`../vif_types`. Note that only the above VIF types are supported by this plugin. diff --git a/os_vif/tests/test_os_vif.py b/os_vif/tests/test_os_vif.py index 0d3fb0a9..90c9016e 100644 --- a/os_vif/tests/test_os_vif.py +++ b/os_vif/tests/test_os_vif.py @@ -146,10 +146,11 @@ class TestOSVIF(base.TestCase): self.assertEqual(info.plugin_info[1].plugin_name, "ovs") vif_info = info.plugin_info[1].vif_info - self.assertEqual(len(vif_info), 3) + self.assertEqual(len(vif_info), 4) self.assertEqual(vif_info[0].vif_object_name, "VIFBridge") self.assertEqual(vif_info[1].vif_object_name, "VIFOpenVSwitch") self.assertEqual(vif_info[2].vif_object_name, "VIFVHostUser") + self.assertEqual(vif_info[3].vif_object_name, "VIFHostDevice") def test_host_info_filtered(self): os_vif.initialize() diff --git a/vif_plug_ovs/exception.py b/vif_plug_ovs/exception.py index 66dc4937..011ca46c 100644 --- a/vif_plug_ovs/exception.py +++ b/vif_plug_ovs/exception.py @@ -26,3 +26,12 @@ class MissingPortProfile(osv_exception.ExceptionBase): class WrongPortProfile(osv_exception.ExceptionBase): msg_fmt = _('Port profile %(profile)s is not a subclass ' 'of VIFPortProfileOpenVSwitch') + + +class RepresentorNotFound(osv_exception.ExceptionBase): + msg_fmt = _('Failed getting representor port for PF %(ifname)s with ' + '%(vf_num)s') + + +class PciDeviceNotFoundById(osv_exception.ExceptionBase): + msg_fmt = _("PCI device %(id)s not found") diff --git a/vif_plug_ovs/linux_net.py b/vif_plug_ovs/linux_net.py index 1459682b..e50e0a38 100644 --- a/vif_plug_ovs/linux_net.py +++ b/vif_plug_ovs/linux_net.py @@ -19,7 +19,10 @@ """Implements vlans, bridges using linux utilities.""" +import errno +import glob import os +import re import sys from oslo_concurrency import processutils @@ -32,6 +35,8 @@ from vif_plug_ovs import privsep LOG = logging.getLogger(__name__) +VIRTFN_RE = re.compile("virtfn(\d+)") + def _ovs_vsctl(args, timeout=None): full_args = ['ovs-vsctl'] @@ -153,7 +158,7 @@ def ensure_bridge(bridge): process_input='1', check_exit_code=[0, 1]) # we bring up the bridge to allow it to switch packets - processutils.execute('ip', 'link', 'set', bridge, 'up') + set_interface_state(bridge, 'up') @privsep.vif_plug.entrypoint @@ -197,6 +202,12 @@ def _set_device_mtu(dev, mtu): check_exit_code=[0, 2, 254]) +@privsep.vif_plug.entrypoint +def set_interface_state(interface_name, port_state): + processutils.execute('ip', 'link', 'set', interface_name, port_state, + check_exit_code=[0, 2, 254]) + + @privsep.vif_plug.entrypoint def _set_mtu_request(dev, mtu, timeout=None): args = ['--', 'set', 'interface', dev, @@ -212,3 +223,84 @@ def _ovs_supports_mtu_requests(timeout=None): ' a column whose name matches "mtu_request"'): return False return True + + +def get_representor_port(pf_ifname, vf_num): + """Get the representor netdevice which is corresponding to the VF. + + This method gets PF interface name and number of VF. It iterates over all + the interfaces under the PF location and looks for interface that has the + VF number in the phys_port_name. That interface is the representor for + the requested VF. + """ + path = "/sys/class/net/%s/subsystem/" % pf_ifname + try: + for device in os.listdir(path): + if device == pf_ifname: + continue + file_name = os.path.join(path, device, 'phys_port_name') + if not os.path.isfile(file_name): + continue + try: + representor_num = open("%s/%s/phys_port_name" % + (path, device)).readline().rstrip() + if int(representor_num) == int(vf_num): + return device + except IOError as e: + # We want to ignore interfaces which we can't read their + # phys_port_name file. + with excutils.save_and_reraise_exception() as ctxt: + if e.errno == errno.EOPNOTSUPP: + ctxt.reraise = False + except ValueError: + # skip representor_num which we can't convert to integer + continue + + except IOError: + pass + raise exception.RepresentorNotFound(ifname=pf_ifname, vf_num=vf_num) + + +def _get_sysfs_netdev_path(pci_addr, pf_interface): + """Get the sysfs path based on the PCI address of the device. + + Assumes a networking device - will not check for the existence of the path. + """ + if pf_interface: + return "/sys/bus/pci/devices/%s/physfn/net" % (pci_addr) + return "/sys/bus/pci/devices/%s/net" % (pci_addr) + + +def get_ifname_by_pci_address(pci_addr, pf_interface=False): + """Get the interface name based on a VF's pci address + + The returned interface name is either the parent PF's or that of the VF + itself based on the argument of pf_interface. + """ + dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface) + try: + dev_info = os.listdir(dev_path) + return dev_info.pop() + except Exception: + raise exception.PciDeviceNotFoundById(id=pci_addr) + + +def get_vf_num_by_pci_address(pci_addr): + """Get the VF number based on a VF's pci address + + A VF is associated with an VF number, which ip link command uses to + configure it. This number can be obtained from the PCI device filesystem. + """ + virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr) + vf_num = None + try: + for vf_path in glob.iglob(virtfns_path): + if re.search(pci_addr, os.readlink(vf_path)): + t = VIRTFN_RE.search(vf_path) + vf_num = t.group(1) + break + except Exception: + pass + if vf_num is None: + raise exception.PciDeviceNotFoundById(id=pci_addr) + return vf_num diff --git a/vif_plug_ovs/ovs.py b/vif_plug_ovs/ovs.py index aede60f9..62e52611 100644 --- a/vif_plug_ovs/ovs.py +++ b/vif_plug_ovs/ovs.py @@ -79,7 +79,11 @@ class OvsPlugin(plugin.PluginBase): objects.host_info.HostVIFInfo( vif_object_name=objects.vif.VIFVHostUser.__name__, min_version="1.0", - max_version="1.0") + max_version="1.0"), + objects.host_info.HostVIFInfo( + vif_object_name=objects.vif.VIFHostDevice.__name__, + min_version="1.0", + max_version="1.0"), ]) def _get_mtu(self, vif): @@ -151,6 +155,17 @@ class OvsPlugin(plugin.PluginBase): constants.OVS_DATAPATH_SYSTEM) self._create_vif_port(vif, vif.id, instance_info) + def _plug_vf_passthrough(self, vif, instance_info): + linux_net.ensure_ovs_bridge( + vif.network.bridge, constants.OVS_DATAPATH_SYSTEM) + pci_slot = vif.dev_address + pf_ifname = linux_net.get_ifname_by_pci_address( + pci_slot, pf_interface=True) + vf_num = linux_net.get_vf_num_by_pci_address(pci_slot) + representor = linux_net.get_representor_port(pf_ifname, vf_num) + linux_net.set_interface_state(representor, 'up') + self._create_vif_port(vif, representor, instance_info) + def plug(self, vif, instance_info): if not hasattr(vif, "port_profile"): raise exception.MissingPortProfile() @@ -172,6 +187,8 @@ class OvsPlugin(plugin.PluginBase): self._plug_vif_windows(vif, instance_info) elif isinstance(vif, objects.vif.VIFVHostUser): self._plug_vhostuser(vif, instance_info) + elif isinstance(vif, objects.vif.VIFHostDevice): + self._plug_vf_passthrough(vif, instance_info) def _unplug_vhostuser(self, vif, instance_info): linux_net.delete_ovs_vif_port(vif.network.bridge, @@ -200,6 +217,16 @@ class OvsPlugin(plugin.PluginBase): linux_net.delete_ovs_vif_port(vif.network.bridge, vif.id, timeout=self.config.ovs_vsctl_timeout) + def _unplug_vf_passthrough(self, vif, instance_info): + """Remove port from OVS.""" + pci_slot = vif.dev_address + pf_ifname = linux_net.get_ifname_by_pci_address(pci_slot, + pf_interface=True) + vf_num = linux_net.get_vf_num_by_pci_address(pci_slot) + representor = linux_net.get_representor_port(pf_ifname, vf_num) + linux_net.delete_ovs_vif_port(vif.network.bridge, representor) + linux_net.set_interface_state(representor, 'down') + def unplug(self, vif, instance_info): if not hasattr(vif, "port_profile"): raise exception.MissingPortProfile() @@ -218,3 +245,5 @@ class OvsPlugin(plugin.PluginBase): self._unplug_vif_windows(vif, instance_info) elif isinstance(vif, objects.vif.VIFVHostUser): self._unplug_vhostuser(vif, instance_info) + elif isinstance(vif, objects.vif.VIFHostDevice): + self._unplug_vf_passthrough(vif, instance_info) diff --git a/vif_plug_ovs/tests/test_linux_net.py b/vif_plug_ovs/tests/test_linux_net.py index 9be794a4..a347bb50 100644 --- a/vif_plug_ovs/tests/test_linux_net.py +++ b/vif_plug_ovs/tests/test_linux_net.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import glob import mock import os.path import testtools @@ -17,6 +18,7 @@ import testtools from oslo_concurrency import processutils from vif_plug_ovs import constants +from vif_plug_ovs import exception from vif_plug_ovs import linux_net from vif_plug_ovs import privsep @@ -34,7 +36,8 @@ class LinuxNetTest(testtools.TestCase): linux_net.ensure_bridge("br0") self.assertEqual(mock_execute.mock_calls, [ - mock.call('ip', 'link', 'set', 'br0', 'up'), + mock.call('ip', 'link', 'set', 'br0', 'up', + check_exit_code=[0, 2, 254]), ]) self.assertEqual(mock_dev_exists.mock_calls, [ mock.call("br0"), @@ -53,7 +56,8 @@ class LinuxNetTest(testtools.TestCase): mock.call('brctl', 'stp', 'br0', "off"), mock.call('tee', '/sys/class/net/br0/bridge/multicast_snooping', check_exit_code=[0, 1], process_input='0'), - mock.call('ip', 'link', 'set', 'br0', 'up'), + mock.call('ip', 'link', 'set', 'br0', 'up', + check_exit_code=[0, 2, 254]), ]) self.assertEqual(mock_dev_exists.mock_calls, [ mock.call("br0") @@ -74,7 +78,8 @@ class LinuxNetTest(testtools.TestCase): check_exit_code=[0, 1], process_input='0'), mock.call('tee', '/proc/sys/net/ipv6/conf/br0/disable_ipv6', check_exit_code=[0, 1], process_input='1'), - mock.call('ip', 'link', 'set', 'br0', 'up'), + mock.call('ip', 'link', 'set', 'br0', 'up', + check_exit_code=[0, 2, 254]), ]) self.assertEqual(mock_dev_exists.mock_calls, [ mock.call("br0") @@ -296,3 +301,114 @@ class LinuxNetTest(testtools.TestCase): result = linux_net._ovs_supports_mtu_requests(timeout=timeout) mock_vsctl.assert_called_with(args, timeout=timeout) self.assertTrue(result) + + @mock.patch('six.moves.builtins.open') + @mock.patch.object(os.path, 'isfile') + @mock.patch.object(os, 'listdir') + def test_get_representor_port(self, mock_listdir, mock_isfile, mock_open): + mock_listdir.return_value = [ + 'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2' + ] + mock_isfile.side_effect = [True, True, True, True, True] + mock_open.return_value.__enter__ = lambda s: s + mock_open.return_value.__exit__ = mock.Mock() + readline_mock = mock_open.return_value.readline + readline_mock.side_effect = ['', '', '1', '2'] + ifname = linux_net.get_representor_port('pf_ifname', '2') + self.assertEqual('rep_vf_2', ifname) + + @mock.patch('six.moves.builtins.open') + @mock.patch.object(os.path, 'isfile') + @mock.patch.object(os, 'listdir') + def test_get_representor_port_not_found( + self, mock_listdir, mock_isfile, mock_open): + mock_listdir.return_value = [ + 'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2' + ] + mock_isfile.side_effect = [True, True, True, True, True] + mock_open.return_value.__enter__ = lambda s: s + mock_open.return_value.__exit__ = mock.Mock() + readline_mock = mock_open.return_value.readline + readline_mock.side_effect = ['', '', '1', '2'] + self.assertRaises( + exception.RepresentorNotFound, + linux_net.get_representor_port, + 'pf_ifname', '3'), + + @mock.patch('six.moves.builtins.open') + @mock.patch.object(os.path, 'isfile') + @mock.patch.object(os, 'listdir') + def test_get_representor_port_exception( + self, mock_listdir, mock_isfile, mock_open): + mock_listdir.return_value = [ + 'pf_ifname', 'vf1_ifname', 'vf2_ifname', 'rep_vf_1', 'rep_vf_2' + ] + mock_isfile.side_effect = [True, True, True, True, True] + mock_open.return_value.__enter__ = lambda s: s + mock_open.return_value.__exit__ = mock.Mock() + readline_mock = mock_open.return_value.readline + readline_mock.side_effect = ['', IOError(), '1', '2'] + self.assertRaises( + exception.RepresentorNotFound, + linux_net.get_representor_port, + 'pf_ifname', '3'), + + @mock.patch.object(os, 'listdir') + def test_physical_function_inferface_name(self, mock_listdir): + mock_listdir.return_value = ['foo', 'bar'] + ifname = linux_net.get_ifname_by_pci_address( + '0000:00:00.1', pf_interface=True) + self.assertEqual(ifname, 'bar') + + @mock.patch.object(os, 'listdir') + def test_virtual_function_inferface_name(self, mock_listdir): + mock_listdir.return_value = ['foo', 'bar'] + ifname = linux_net.get_ifname_by_pci_address( + '0000:00:00.1', pf_interface=False) + self.assertEqual(ifname, 'bar') + + @mock.patch.object(os, 'listdir') + def test_get_ifname_by_pci_address_exception(self, mock_listdir): + mock_listdir.side_effect = OSError('No such file or directory') + self.assertRaises( + exception.PciDeviceNotFoundById, + linux_net.get_ifname_by_pci_address, + '0000:00:00.1' + ) + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + def test_vf_number_found(self, mock_iglob, mock_readlink): + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3', + ] + mock_readlink.return_value = '../../0000:00:00.1' + vf_num = linux_net.get_vf_num_by_pci_address('0000:00:00.1') + self.assertEqual(vf_num, '3') + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + def test_vf_number_not_found(self, mock_iglob, mock_readlink): + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3', + ] + mock_readlink.return_value = '../../0000:00:00.2' + self.assertRaises( + exception.PciDeviceNotFoundById, + linux_net.get_vf_num_by_pci_address, + '0000:00:00.1' + ) + + @mock.patch.object(os, 'readlink') + @mock.patch.object(glob, 'iglob') + def test_get_vf_num_by_pci_address_exception( + self, mock_iglob, mock_readlink): + mock_iglob.return_value = [ + '/sys/bus/pci/devices/0000:00:00.1/physfn/virtfn3', + ] + mock_readlink.side_effect = OSError('No such file or directory') + self.assertRaises( + exception.PciDeviceNotFoundById, + linux_net.get_vf_num_by_pci_address, + '0000:00:00.1' + ) diff --git a/vif_plug_ovs/tests/test_plugin.py b/vif_plug_ovs/tests/test_plugin.py index 590b8319..4f5baf1e 100644 --- a/vif_plug_ovs/tests/test_plugin.py +++ b/vif_plug_ovs/tests/test_plugin.py @@ -14,6 +14,7 @@ import mock import testtools from os_vif import objects +from os_vif.objects import fields from vif_plug_ovs import constants from vif_plug_ovs import linux_net @@ -88,6 +89,15 @@ class PluginTest(testtools.TestCase): mode='server', # qemu server mode <=> ovs client mode port_profile=self.profile_ovs) + self.vif_ovs_vf_passthrough = objects.vif.VIFHostDevice( + id='b679325f-ca89-4ee0-a8be-6db1409b69ea', + address='ca:fe:de:ad:be:ef', + network=self.network_ovs, + dev_type=fields.VIFVIFHostDeviceDevType.ETHERNET, + dev_address='0002:24:12.3', + bridge_name='br-int', + port_profile=self.profile_ovs) + self.instance = objects.instance_info.InstanceInfo( name='demo', uuid='f0000000-0000-0000-0000-000000000001') @@ -133,6 +143,7 @@ class PluginTest(testtools.TestCase): ensure_ovs_bridge.assert_called_once_with( self.vif_ovs.network.bridge, constants.OVS_DATAPATH_SYSTEM) + @mock.patch.object(linux_net, 'set_interface_state') @mock.patch.object(linux_net, 'ensure_ovs_bridge') @mock.patch.object(ovs.OvsPlugin, '_update_vif_port') @mock.patch.object(ovs.OvsPlugin, '_create_vif_port') @@ -145,7 +156,8 @@ class PluginTest(testtools.TestCase): def test_plug_ovs_bridge(self, mock_sys, ensure_bridge, device_exists, create_veth_pair, update_veth_pair, add_bridge_port, _create_vif_port, - _update_vif_port, ensure_ovs_bridge): + _update_vif_port, ensure_ovs_bridge, + set_interface_state): calls = { 'device_exists': [mock.call('qvob679325f-ca')], 'create_veth_pair': [mock.call('qvbb679325f-ca', @@ -155,6 +167,8 @@ class PluginTest(testtools.TestCase): 'qvob679325f-ca', 1500)], 'ensure_bridge': [mock.call('qbrvif-xxx-yyy')], + 'set_interface_state': [mock.call('qbrvif-xxx-yyy', + 'up')], 'add_bridge_port': [mock.call('qbrvif-xxx-yyy', 'qvbb679325f-ca')], '_update_vif_port': [mock.call(self.vif_ovs_hybrid, @@ -309,3 +323,79 @@ class PluginTest(testtools.TestCase): plugin = ovs.OvsPlugin.load("ovs") plugin.unplug(self.vif_vhostuser, self.instance) delete_ovs_vif_port.assert_has_calls(calls['delete_ovs_vif_port']) + + @mock.patch.object(linux_net, 'ensure_ovs_bridge') + @mock.patch.object(linux_net, 'get_ifname_by_pci_address') + @mock.patch.object(linux_net, 'get_vf_num_by_pci_address') + @mock.patch.object(linux_net, 'get_representor_port') + @mock.patch.object(linux_net, 'set_interface_state') + @mock.patch.object(ovs.OvsPlugin, '_create_vif_port') + def test_plug_ovs_vf_passthrough(self, _create_vif_port, + set_interface_state, + get_representor_port, + get_vf_num_by_pci_address, + get_ifname_by_pci_address, + ensure_ovs_bridge): + + get_ifname_by_pci_address.return_value = 'eth0' + get_vf_num_by_pci_address.return_value = '2' + get_representor_port.return_value = 'eth0_2' + calls = { + + 'ensure_ovs_bridge': [mock.call('br0', + constants.OVS_DATAPATH_SYSTEM)], + 'get_ifname_by_pci_address': [mock.call('0002:24:12.3', + pf_interface=True)], + 'get_vf_num_by_pci_address': [mock.call('0002:24:12.3')], + 'get_representor_port': [mock.call('eth0', '2')], + 'set_interface_state': [mock.call('eth0_2', 'up')], + '_create_vif_port': [mock.call( + self.vif_ovs_vf_passthrough, 'eth0_2', + self.instance)] + } + + plugin = ovs.OvsPlugin.load("ovs") + plugin.plug(self.vif_ovs_vf_passthrough, self.instance) + ensure_ovs_bridge.assert_has_calls(calls['ensure_ovs_bridge']) + get_ifname_by_pci_address.assert_has_calls( + calls['get_ifname_by_pci_address']) + get_vf_num_by_pci_address.assert_has_calls( + calls['get_vf_num_by_pci_address']) + get_representor_port.assert_has_calls( + calls['get_representor_port']) + set_interface_state.assert_has_calls(calls['set_interface_state']) + _create_vif_port.assert_has_calls(calls['_create_vif_port']) + + @mock.patch.object(linux_net, 'get_ifname_by_pci_address') + @mock.patch.object(linux_net, 'get_vf_num_by_pci_address') + @mock.patch.object(linux_net, 'get_representor_port') + @mock.patch.object(linux_net, 'set_interface_state') + @mock.patch.object(linux_net, 'delete_ovs_vif_port') + def test_unplug_ovs_vf_passthrough(self, delete_ovs_vif_port, + set_interface_state, + get_representor_port, + get_vf_num_by_pci_address, + get_ifname_by_pci_address): + calls = { + + 'get_ifname_by_pci_address': [mock.call('0002:24:12.3', + pf_interface=True)], + 'get_vf_num_by_pci_address': [mock.call('0002:24:12.3')], + 'get_representor_port': [mock.call('eth0', '2')], + 'set_interface_state': [mock.call('eth0_2', 'down')], + 'delete_ovs_vif_port': [mock.call('br0', 'eth0_2')] + } + + get_ifname_by_pci_address.return_value = 'eth0' + get_vf_num_by_pci_address.return_value = '2' + get_representor_port.return_value = 'eth0_2' + plugin = ovs.OvsPlugin.load("ovs") + plugin.unplug(self.vif_ovs_vf_passthrough, self.instance) + get_ifname_by_pci_address.assert_has_calls( + calls['get_ifname_by_pci_address']) + get_vf_num_by_pci_address.assert_has_calls( + calls['get_vf_num_by_pci_address']) + get_representor_port.assert_has_calls( + calls['get_representor_port']) + delete_ovs_vif_port.assert_has_calls(calls['delete_ovs_vif_port']) + set_interface_state.assert_has_calls(calls['set_interface_state'])