From e6530bc2bb09a14e5820cdce74e4f7fd8e8d75e6 Mon Sep 17 00:00:00 2001 From: Georg Kunz Date: Fri, 21 Dec 2018 14:04:37 +0100 Subject: [PATCH] Extended OVS chart with support for DPDK Extending the Openvswitch chart with support for DPDK. In order to enable DPDK support, set the dpdk:enabled option to true in value.yaml. Prerequisites for successfully running OVS with DPDK: the host OS must to have hugepages enabled. Co-Authored-By: Rihab Banday Change-Id: I9649832511ba7c7ba7c391555d60171ef9264110 --- .../bin/_openvswitch-vswitchd.sh.tpl | 27 ++++++++- .../templates/daemonset-ovs-vswitchd.yaml | 59 +++++++++++++++++++ openvswitch/values.yaml | 19 +++++- ...rameter-to-ovs-chart-41d2b05b79300a31.yaml | 11 ++++ ...ge-default-ovs-image-c1e24787f1b03170.yaml | 7 +++ 5 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml create mode 100644 releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml diff --git a/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl index 373e0162e..70151a1e1 100644 --- a/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl +++ b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl @@ -22,6 +22,11 @@ COMMAND="${@:-start}" OVS_SOCKET=/run/openvswitch/db.sock OVS_PID=/run/openvswitch/ovs-vswitchd.pid +# Create vhostuser directory and grant nova user (default UID 42424) access +# permissions. +mkdir -p /run/openvswitch/vhostuser +chown {{ .Values.pod.user.nova.uid }}.{{ .Values.pod.user.nova.uid }} /run/openvswitch/vhostuser + function start () { t=0 while [ ! -e "${OVS_SOCKET}" ] ; do @@ -34,7 +39,27 @@ function start () { fi done - ovs-vsctl --no-wait show + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait show + +{{- if .Values.conf.dpdk.enabled }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-hugepage-dir={{ .Values.conf.dpdk.hugepages_mountpath | quote }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-socket-mem={{ .Values.conf.dpdk.socket_memory | quote }} + +{{- if .Values.conf.dpdk.mem_channels }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-mem-channels={{ .Values.conf.dpdk.mem_channels | quote }} +{{- end }} + +{{- if .Values.conf.dpdk.pmd_cpu_mask }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:pmd-cpu-mask={{ .Values.conf.dpdk.pmd_cpu_mask | quote }} +{{- end }} + +{{- if .Values.conf.dpdk.lcore_mask }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask={{ .Values.conf.dpdk.lcore_mask | quote }} +{{- end }} + + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:vhost-sock-dir="vhostuser" + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-init=true +{{- end }} exec /usr/sbin/ovs-vswitchd unix:${OVS_SOCKET} \ -vconsole:emer \ diff --git a/openvswitch/templates/daemonset-ovs-vswitchd.yaml b/openvswitch/templates/daemonset-ovs-vswitchd.yaml index 0c337fb89..a609030a6 100644 --- a/openvswitch/templates/daemonset-ovs-vswitchd.yaml +++ b/openvswitch/templates/daemonset-ovs-vswitchd.yaml @@ -68,8 +68,19 @@ spec: readOnly: true containers: - name: openvswitch-vswitchd +{{- if .Values.conf.dpdk.enabled }} +{{/* Run the container in priviledged mode due to the need for root +permissions when using the uio_pci_generic driver. */}} +{{- $_ := set $envAll.Values.pod.security_context.openvswitch_vswitchd.container.vswitchd "privileged" true -}} +{{- end }} {{ tuple $envAll "openvswitch_vswitchd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ dict "envAll" $envAll "application" "openvswitch_vswitchd" "container" "vswitchd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{- if .Values.conf.dpdk.enabled }} +{{/* When running with DPDK, we need to specify the type and amount of hugepages. +The following line enables resource handling in general, but the type and amount +of hugepages must still be defined in the values.yaml.*/}} +{{ $_ := set $envAll.Values.pod.resources "enabled" true }} +{{- end }} {{ tuple $envAll $envAll.Values.pod.resources.ovs.vswitchd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} # ensures this container can speak to the ovs database # successfully before its marked as ready @@ -104,6 +115,22 @@ spec: readOnly: true - name: run mountPath: /run +{{- if .Values.conf.dpdk.enabled }} + - name: hugepages + mountPath: {{ .Values.conf.dpdk.hugepages_mountpath | quote }} + - name: pci-devices + mountPath: /sys/bus/pci/devices + - name: huge-pages-kernel + mountPath: /sys/kernel/mm/hugepages + - name: node-devices + mountPath: /sys/devices/system/node + - name: modules + mountPath: /lib/modules + - name: devs + mountPath: /dev + - name: pci-drivers + mountPath: /sys/bus/pci/drivers +{{- end }} volumes: - name: pod-tmp emptyDir: {} @@ -114,7 +141,39 @@ spec: - name: run hostPath: path: /run + type: Directory - name: host-rootfs hostPath: path: / + type: Directory +{{- if .Values.conf.dpdk.enabled }} + - name: devs + hostPath: + path: /dev + type: Directory + - name: pci-devices + hostPath: + path: /sys/bus/pci/devices + type: Directory + - name: huge-pages-kernel + hostPath: + path: /sys/kernel/mm/hugepages + type: Directory + - name: node-devices + hostPath: + path: /sys/devices/system/node + type: Directory + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: pci-drivers + hostPath: + path: /sys/bus/pci/drivers + type: Directory + - name: hugepages + hostPath: + path: {{ .Values.conf.dpdk.hugepages_mountpath | quote }} + type: Directory +{{- end }} {{- end }} diff --git a/openvswitch/values.yaml b/openvswitch/values.yaml index bb69740b5..52ef70bdf 100644 --- a/openvswitch/values.yaml +++ b/openvswitch/values.yaml @@ -21,8 +21,8 @@ release_group: null images: tags: - openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-debian - openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-debian + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_bionic + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_bionic dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" @@ -96,6 +96,9 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + # set resources to enabled and specify one of the following when using dpdk + # hugepages-1Gi: "1Gi" + # hugepages-2Mi: "512Mi" jobs: image_repo_sync: requests: @@ -104,6 +107,9 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + user: + nova: + uid: 42424 endpoints: cluster_domain_suffix: cluster.local @@ -152,3 +158,12 @@ manifests: conf: openvswitch_db_server: ptcp_port: null + dpdk: + enabled: false + socket_memory: 1024 + hugepages_mountpath: /dev/hugepages + # optional parameters for tuning the OVS config + # in alignment with the available hardware resources + # mem_channels: 4 + # lcore_mask: 0x1 + # pmd_cpu_mask: 0x4 diff --git a/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml b/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml new file mode 100644 index 000000000..cae56c16d --- /dev/null +++ b/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml @@ -0,0 +1,11 @@ +--- +other: + - | + When running openvswitch (OVS) with DPDK enabled, vhost-user sockets are + used to connect VMs to OVS. nova-compute needs access to those sockets in + order to plug them into OVS. For this reason, the directory containing + vhost-user sockets must have proper permissions. The openvswitch chart now + sets ownership of this directory to the UID of the nova user. The OVS chart + uses the same default as the Nova chart (42424). However, if the Nova UID + is changed in the Nova chart in a particular deployment, it also needs to + be changed in the OVS chart correspondingly if DPDK is used. diff --git a/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml b/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml new file mode 100644 index 000000000..698adbd36 --- /dev/null +++ b/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml @@ -0,0 +1,7 @@ +--- +other: + - | + The default image used by the openvswitch chart has been changed from a + a Debian based image including a source build of openvswitch v2.8.1 to an + Ubuntu Bionic based image including a distribution provided build of + openvswitch v2.9.2.