Extended OVS chart with support for DPDK

Extending the Openvswitch chart with support for DPDK. In order to
enable DPDK support, set the dpdk:enabled option to true in value.yaml.
Prerequisites for successfully running OVS with DPDK: the host OS must
to have hugepages enabled.

Co-Authored-By: Rihab Banday <rihab.banday@ericsson.com>

Change-Id: I9649832511ba7c7ba7c391555d60171ef9264110

openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl

@@ -22,6 +22,11 @@ COMMAND="${@:-start}"
 OVS_SOCKET=/run/openvswitch/db.sock
 OVS_PID=/run/openvswitch/ovs-vswitchd.pid
 
+# Create vhostuser directory and grant nova user (default UID 42424) access
+# permissions.
+mkdir -p /run/openvswitch/vhostuser
+chown {{ .Values.pod.user.nova.uid }}.{{ .Values.pod.user.nova.uid }} /run/openvswitch/vhostuser
+
 function start () {
   t=0
   while [ ! -e "${OVS_SOCKET}" ] ; do
@@ -34,7 +39,27 @@ function start () {
       fi
   done
 
-  ovs-vsctl --no-wait show
+  ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait show
+
+{{- if .Values.conf.dpdk.enabled }}
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-hugepage-dir={{ .Values.conf.dpdk.hugepages_mountpath | quote }}
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-socket-mem={{ .Values.conf.dpdk.socket_memory | quote }}
+
+{{- if .Values.conf.dpdk.mem_channels }}
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-mem-channels={{ .Values.conf.dpdk.mem_channels | quote }}
+{{- end }}
+
+{{- if .Values.conf.dpdk.pmd_cpu_mask }}
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:pmd-cpu-mask={{ .Values.conf.dpdk.pmd_cpu_mask | quote }}
+{{- end }}
+
+{{- if .Values.conf.dpdk.lcore_mask }}
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask={{ .Values.conf.dpdk.lcore_mask | quote }}
+{{- end }}
+
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:vhost-sock-dir="vhostuser"
+    ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-init=true
+{{- end }}
 
   exec /usr/sbin/ovs-vswitchd unix:${OVS_SOCKET} \
           -vconsole:emer \

openvswitch/templates/daemonset-ovs-vswitchd.yaml

@@ -68,8 +68,19 @@ spec:
               readOnly: true
       containers:
         - name: openvswitch-vswitchd
+{{- if .Values.conf.dpdk.enabled }}
+{{/* Run the container in priviledged mode due to the need for root
+permissions when using the uio_pci_generic driver. */}}
+{{- $_ := set $envAll.Values.pod.security_context.openvswitch_vswitchd.container.vswitchd "privileged" true -}}
+{{- end }}
 {{ tuple $envAll "openvswitch_vswitchd" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ dict "envAll" $envAll "application" "openvswitch_vswitchd" "container" "vswitchd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{- if .Values.conf.dpdk.enabled }}
+{{/* When running with DPDK, we need to specify the type and amount of hugepages.
+The following line enables resource handling in general, but the type and amount
+of hugepages must still be defined in the values.yaml.*/}}
+{{ $_ := set $envAll.Values.pod.resources "enabled" true }}
+{{- end }}
 {{ tuple $envAll $envAll.Values.pod.resources.ovs.vswitchd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           # ensures this container can speak to the ovs database
           # successfully before its marked as ready
@@ -104,6 +115,22 @@ spec:
               readOnly: true
             - name: run
               mountPath: /run
+{{- if .Values.conf.dpdk.enabled }}
+            - name: hugepages
+              mountPath: {{ .Values.conf.dpdk.hugepages_mountpath | quote }}
+            - name: pci-devices
+              mountPath: /sys/bus/pci/devices
+            - name: huge-pages-kernel
+              mountPath: /sys/kernel/mm/hugepages
+            - name: node-devices
+              mountPath: /sys/devices/system/node
+            - name: modules
+              mountPath: /lib/modules
+            - name: devs
+              mountPath: /dev
+            - name: pci-drivers
+              mountPath: /sys/bus/pci/drivers
+{{- end }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -114,7 +141,39 @@ spec:
         - name: run
           hostPath:
             path: /run
+            type: Directory
         - name: host-rootfs
           hostPath:
             path: /
+            type: Directory
+{{- if .Values.conf.dpdk.enabled }}
+        - name: devs
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: pci-devices
+          hostPath:
+            path: /sys/bus/pci/devices
+            type: Directory
+        - name: huge-pages-kernel
+          hostPath:
+            path: /sys/kernel/mm/hugepages
+            type: Directory
+        - name: node-devices
+          hostPath:
+            path: /sys/devices/system/node
+            type: Directory
+        - name: modules
+          hostPath:
+            path: /lib/modules
+            type: Directory
+        - name: pci-drivers
+          hostPath:
+            path: /sys/bus/pci/drivers
+            type: Directory
+        - name: hugepages
+          hostPath:
+            path: {{ .Values.conf.dpdk.hugepages_mountpath | quote }}
+            type: Directory
+{{- end }}
 {{- end }}

openvswitch/values.yaml

@@ -21,8 +21,8 @@ release_group: null
 
 images:
   tags:
-    openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-debian
+    openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_bionic
-    openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-debian
+    openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_bionic
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: "IfNotPresent"
@@ -96,6 +96,9 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+          # set resources to enabled and specify one of the following when using dpdk
+          # hugepages-1Gi: "1Gi"
+          # hugepages-2Mi: "512Mi"
     jobs:
       image_repo_sync:
         requests:
@@ -104,6 +107,9 @@ pod:
         limits:
           memory: "1024Mi"
           cpu: "2000m"
+  user:
+    nova:
+      uid: 42424
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -152,3 +158,12 @@ manifests:
 conf:
   openvswitch_db_server:
     ptcp_port: null
+  dpdk:
+    enabled: false
+    socket_memory: 1024
+    hugepages_mountpath: /dev/hugepages
+    # optional parameters for tuning the OVS config
+    # in alignment with the available hardware resources
+    # mem_channels: 4
+    # lcore_mask: 0x1
+    # pmd_cpu_mask: 0x4

releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml

@@ -0,0 +1,11 @@
+---
+other:
+  - |
+    When running openvswitch (OVS) with DPDK enabled, vhost-user sockets are
+    used to connect VMs to OVS. nova-compute needs access to those sockets in
+    order to plug them into OVS. For this reason, the directory containing
+    vhost-user sockets must have proper permissions. The openvswitch chart now
+    sets ownership of this directory to the UID of the nova user. The OVS chart
+    uses the same default as the Nova chart (42424). However, if the Nova UID
+    is changed in the Nova chart in a particular deployment, it also needs to
+    be changed in the OVS chart correspondingly if DPDK is used.

releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml

@@ -0,0 +1,7 @@
+---
+other:
+  - |
+    The default image used by the openvswitch chart has been changed from a
+    a Debian based image including a source build of openvswitch v2.8.1 to an
+    Ubuntu Bionic based image including a distribution provided build of
+    openvswitch v2.9.2.