Enable support for hardware offload

Enable support for use of hardware offload via OVS; this requires
OpenStack Stein or later in conjunction with the latest HWE kernel
for Ubuntu 18.04 LTS.

Change-Id: I4ce47b1712e79bfbed9ac708cc521840b3709724

config.yaml

@@ -238,6 +238,23 @@ options:
       NOTE: Changing this value will disrupt networking on all SR-IOV capable
       interfaces for blanket configuration or listed interfaces when per-device
       configuration is used.
+  enable-hardware-offload:
+    type: boolean
+    default: false
+    description: |
+      Enable support for hardware offload of flows from Open vSwitch to supported
+      network adapters.  This requires use of OpenStack Stein or later and has
+      only been tested on Mellanox ConnectX 5 adapters.
+      .
+      Enabling this option will make use of the sriov-numvfs option to configure
+      the VF functions of the physical network adapters detected in each unit.
+      .
+      Enabling this option also requires that the firewall-driver option be
+      set to 'openvswitch'; this will allow security groups to be applied to
+      hardware offloaded ports (note that this feature is currently not
+      supported by OVS or the Linux kernel).
+      .
+      This option must not be enabled with either enable-sriov or enable-dpdk.
   networking-tools-source:
     type: string
     default: ppa:openstack-charmers/networking-tools

hooks/neutron_ovs_utils.py

@@ -51,6 +51,7 @@ from charmhelpers.core.hookenv import (
     DEBUG,
     log,
     status_set,
+    ERROR,
 )
 from charmhelpers.contrib.openstack.neutron import (
     parse_bridge_mappings,
@@ -288,6 +289,9 @@ def install_packages():
     if use_dpdk():
         enable_ovs_dpdk()
 
+    if use_hw_offload():
+        enable_hw_offload()
+
     # NOTE(tpsilva): if we're using openvswitch driver, we need to explicitly
     #                load the nf_conntrack_ipv4/6 module, since it won't be
     #                loaded automatically in some cases. LP#1834213
@@ -356,6 +360,11 @@ def determine_packages():
             pkgs.append('neutron-plugin-sriov-agent')
         pkgs.append('sriov-netplan-shim')
 
+    if use_hw_offload():
+        pkgs.append('mlnx-switchdev-mode')
+        if 'sriov-netplan-shim' not in pkgs:
+            pkgs.append('sriov-netplan-shim')
+
     if cmp_release >= 'rocky':
         pkgs = [p for p in pkgs if not p.startswith('python-')]
         pkgs.extend(PY3_PACKAGES)
@@ -545,6 +554,19 @@ def enable_ovs_dpdk():
         service_restart('openvswitch-switch')
 
 
+def enable_hw_offload():
+    '''Enable hardware offload support in Open vSwitch'''
+    values_changed = [
+        set_Open_vSwitch_column_value('other_config:hw-offload',
+                                      'true'),
+        set_Open_vSwitch_column_value('other_config:max-idle',
+                                      '30000')
+    ]
+    if ((values_changed and any(values_changed)) and
+            not is_unit_paused_set()):
+        service_restart('openvswitch-switch')
+
+
 def install_tmpfilesd():
     '''Install systemd-tmpfiles configuration for ovs vhost-user sockets'''
     # NOTE(jamespage): Only do this if libvirt is actually installed
@@ -604,7 +626,14 @@ def configure_ovs():
                     else:
                         add_ovsbridge_linuxbridge(br, port)
 
-    if use_dpdk():
+    # NOTE(jamespage):
+    # hw-offload and dpdk are mutually exclusive so log and error
+    # and skip any subsequent DPDK configuration
+    if use_dpdk() and use_hw_offload():
+        log('DPDK and Hardware offload are mutually exclusive, '
+            'please disable enable-dpdk or enable-hardware-offload',
+            level=ERROR)
+    elif use_dpdk():
         log('Configuring bridges with DPDK', level=DEBUG)
         global_mtu = (
             neutron_ovs_context.NeutronAPIContext()()['global_physnet_mtu'])
@@ -809,7 +838,11 @@ def configure_sriov():
             yaml.dump(interfaces_yaml, _conf)
         return numvfs_changed
 
-    if not enable_sriov():
+    # NOTE(jamespage)
+    # Hardware offload makes use of SR-IOV VF representor ports so
+    # makes use of the general SR-IOV configuration support in this
+    # charm
+    if not (enable_sriov() or use_hw_offload()):
         return
 
     # Tidy up any prior installation of obsolete sriov startup
@@ -821,7 +854,7 @@ def configure_sriov():
     # Trigger remote restart in parent application
     remote_restart('neutron-plugin', 'nova-compute')
 
-    if numvfs_changed:
+    if not use_hw_offload() and numvfs_changed:
         # Restart of SRIOV agent is required after changes to system runtime
         # VF configuration
         cmp_release = CompareOpenStackReleases(
@@ -831,6 +864,9 @@ def configure_sriov():
         else:
             service_restart('neutron-plugin-sriov-agent')
 
+    if use_hw_offload() and numvfs_changed:
+        service_restart('mlnx-switchdev-mode')
+
 
 def get_shared_secret():
     ctxt = neutron_ovs_context.SharedSecretContext()()
@@ -866,6 +902,19 @@ def use_dpdk():
     return (cmp_release >= 'mitaka' and config('enable-dpdk'))
 
 
+def use_hw_offload():
+    '''
+    Determine whether OVS hardware offload should be used
+
+    :returns: boolean indicating whether hardware offload should be enabled
+    :rtype: bool
+    '''
+    cmp_release = CompareOpenStackReleases(
+        os_release('neutron-common')
+    )
+    return (cmp_release >= 'stein' and config('enable-hardware-offload'))
+
+
 def ovs_has_late_dpdk_init():
     ''' OVS 2.6.0 introduces late initialization '''
     import apt_pkg

unit_tests/test_neutron_ovs_utils.py

@@ -74,6 +74,7 @@ TO_PATCH = [
     'init_is_systemd',
     'modprobe',
     'is_container',
+    'is_unit_paused_set',
 ]
 
 head_pkg = 'linux-headers-3.15.0-5-generic'
@@ -182,6 +183,27 @@ class TestNeutronOVSUtils(CharmTestCase):
             call(self.filter_installed_packages(), fatal=True),
         ])
 
+    @patch.object(nutils, 'use_hw_offload')
+    @patch.object(nutils, 'enable_hw_offload')
+    @patch.object(nutils, 'determine_packages')
+    def test_install_packages_hwoffload(self, _determine_packages,
+                                        _enable_hw_offload,
+                                        _use_hw_offload):
+        self.os_release.return_value = 'stein'
+        _determine_packages.return_value = 'randompkg'
+        _use_hw_offload.return_value = True
+        self.determine_dkms_package.return_value = \
+            ['openvswitch-datapath-dkms']
+        self.headers_package.return_value = 'linux-headers-foobar'
+        nutils.install_packages()
+        self.apt_update.assert_called_with()
+        self.apt_install.assert_has_calls([
+            call(['linux-headers-foobar',
+                  'openvswitch-datapath-dkms'], fatal=True),
+            call(self.filter_installed_packages(), fatal=True),
+        ])
+        _enable_hw_offload.assert_called_once_with()
+
     @patch.object(nutils, 'use_l3ha')
     @patch.object(nutils, 'use_dvr')
     @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release')
@@ -394,6 +416,33 @@ class TestNeutronOVSUtils(CharmTestCase):
         ]
         self.assertEqual(pkg_list, expect)
 
+    @patch.object(nutils, 'use_hw_offload')
+    @patch.object(nutils, 'use_l3ha')
+    @patch.object(nutils, 'use_dvr')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'os_release')
+    @patch.object(charmhelpers.contrib.openstack.neutron, 'headers_package')
+    def test_determine_pkgs_hardware_offload(self, _head_pkgs, _os_rel,
+                                             _use_dvr, _use_l3ha,
+                                             _use_hw_offload):
+        self.test_config.set('enable-local-dhcp-and-metadata', False)
+        self.test_config.set('enable-hardware-offload', True)
+        _use_hw_offload.return_value = True
+        _use_dvr.return_value = False
+        _use_l3ha.return_value = False
+        _os_rel.return_value = 'stein'
+        self.os_release.return_value = 'stein'
+        _head_pkgs.return_value = head_pkg
+        pkg_list = nutils.determine_packages()
+        expect = [
+            head_pkg,
+            'neutron-openvswitch-agent',
+            'mlnx-switchdev-mode',
+            'sriov-netplan-shim',
+            'python3-neutron',
+            'python3-zmq',
+        ]
+        self.assertEqual(pkg_list, expect)
+
     @patch.object(nutils, 'use_dvr')
     def test_register_configs(self, _use_dvr):
         class _mock_OSConfigRenderer():
@@ -764,6 +813,7 @@ class TestNeutronOVSUtils(CharmTestCase):
                                         2, _late_init), 1500)],
                 any_order=True)
 
+    @patch.object(nutils, 'use_hw_offload', return_value=False)
     @patch.object(neutron_ovs_context, 'NeutronAPIContext')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges')
@@ -772,7 +822,8 @@ class TestNeutronOVSUtils(CharmTestCase):
     def test_configure_ovs_dpdk(self, mock_config, _use_dvr,
                                 _resolve_dpdk_bridges,
                                 _resolve_dpdk_bonds,
-                                _NeutronAPIContext):
+                                _NeutronAPIContext,
+                                _use_hw_offload):
         _NeutronAPIContext.return_value = DummyContext(
             return_value={'global_physnet_mtu': 1500})
         return self._run_configure_ovs_dpdk(mock_config, _use_dvr,
@@ -781,6 +832,7 @@ class TestNeutronOVSUtils(CharmTestCase):
                                             _late_init=False,
                                             _test_bonds=False)
 
+    @patch.object(nutils, 'use_hw_offload', return_value=False)
     @patch.object(neutron_ovs_context, 'NeutronAPIContext')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges')
@@ -789,7 +841,8 @@ class TestNeutronOVSUtils(CharmTestCase):
     def test_configure_ovs_dpdk_late_init(self, mock_config, _use_dvr,
                                           _resolve_dpdk_bridges,
                                           _resolve_dpdk_bonds,
-                                          _NeutronAPIContext):
+                                          _NeutronAPIContext,
+                                          _use_hw_offload):
         _NeutronAPIContext.return_value = DummyContext(
             return_value={'global_physnet_mtu': 1500})
         return self._run_configure_ovs_dpdk(mock_config, _use_dvr,
@@ -798,6 +851,7 @@ class TestNeutronOVSUtils(CharmTestCase):
                                             _late_init=True,
                                             _test_bonds=False)
 
+    @patch.object(nutils, 'use_hw_offload', return_value=False)
     @patch.object(neutron_ovs_context, 'NeutronAPIContext')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bonds')
     @patch.object(neutron_ovs_context, 'resolve_dpdk_bridges')
@@ -806,7 +860,8 @@ class TestNeutronOVSUtils(CharmTestCase):
     def test_configure_ovs_dpdk_late_init_bonds(self, mock_config, _use_dvr,
                                                 _resolve_dpdk_bridges,
                                                 _resolve_dpdk_bonds,
-                                                _NeutronAPIContext):
+                                                _NeutronAPIContext,
+                                                _use_hw_offload):
         _NeutronAPIContext.return_value = DummyContext(
             return_value={'global_physnet_mtu': 1500})
         return self._run_configure_ovs_dpdk(mock_config, _use_dvr,
@@ -1170,6 +1225,54 @@ class TestNeutronOVSUtils(CharmTestCase):
         _kv().get.return_value = True
         self.assertEquals(nutils.use_fqdn_hint(), True)
 
+    def test_use_hw_offload_rocky(self):
+        self.os_release.return_value = 'rocky'
+        self.test_config.set('enable-hardware-offload', True)
+        self.assertFalse(nutils.use_hw_offload())
+
+    def test_use_hw_offload_stein(self):
+        self.os_release.return_value = 'stein'
+        self.test_config.set('enable-hardware-offload', True)
+        self.assertTrue(nutils.use_hw_offload())
+
+    def test_use_hw_offload_disabled(self):
+        self.os_release.return_value = 'stein'
+        self.test_config.set('enable-hardware-offload', False)
+        self.assertFalse(nutils.use_hw_offload())
+
+    @patch.object(nutils, 'set_Open_vSwitch_column_value')
+    def test_enable_hw_offload(self, _ovs_set):
+        _ovs_set.return_value = True
+        self.is_unit_paused_set.return_value = False
+        nutils.enable_hw_offload()
+        _ovs_set.assert_has_calls([
+            call('other_config:hw-offload', 'true'),
+            call('other_config:max-idle', '30000'),
+        ])
+        self.service_restart.assert_called_once_with('openvswitch-switch')
+
+    @patch.object(nutils, 'set_Open_vSwitch_column_value')
+    def test_enable_hw_offload_unit_paused(self, _ovs_set):
+        _ovs_set.return_value = True
+        self.is_unit_paused_set.return_value = True
+        nutils.enable_hw_offload()
+        _ovs_set.assert_has_calls([
+            call('other_config:hw-offload', 'true'),
+            call('other_config:max-idle', '30000'),
+        ])
+        self.service_restart.assert_not_called()
+
+    @patch.object(nutils, 'set_Open_vSwitch_column_value')
+    def test_enable_hw_offload_no_changes(self, _ovs_set):
+        _ovs_set.return_value = False
+        self.is_unit_paused_set.return_value = False
+        nutils.enable_hw_offload()
+        _ovs_set.assert_has_calls([
+            call('other_config:hw-offload', 'true'),
+            call('other_config:max-idle', '30000'),
+        ])
+        self.service_restart.assert_not_called()
+
 
 class TestDPDKBridgeBondMap(CharmTestCase):