From e354bef7bb4fafc31789efb149fb0b3fc69fd8f4 Mon Sep 17 00:00:00 2001
From: Deepak Tiwari <deepak.tiwari@aricent.com>
Date: Thu, 29 Aug 2019 13:57:50 -0500
Subject: [PATCH] ovs-dpdk: create separate cgroup for openvswitch-vswitchd
 pods

Setup Cgroup to use to break out of Kubernetes defined groups for ovs-dpdk pods.
All the cores on the server are added to the cpuset, pmd_cpu_mask and lcore_mask
will choose the right ones for ovs-dpdk from all the cores.

Co-Authored-By: Phil Sphicas <ps3910@att.com>

Change-Id: Ia840647e3fc09480b826b3075b2585daefa638b3
---
 .../bin/_openvswitch-vswitchd.sh.tpl          | 20 +++++++++++++++++++
 .../templates/daemonset-ovs-vswitchd.yaml     |  5 +++++
 2 files changed, 25 insertions(+)

diff --git a/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl
index f50cc4e4e..9bb341f7d 100644
--- a/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl
+++ b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl
@@ -53,14 +53,34 @@ function start () {
 
 {{- if .Values.conf.ovs_dpdk.pmd_cpu_mask }}
     ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:pmd-cpu-mask={{ .Values.conf.ovs_dpdk.pmd_cpu_mask | quote }}
+    PMD_CPU_MASK={{ .Values.conf.ovs_dpdk.pmd_cpu_mask | quote }}
 {{- end }}
 
 {{- if .Values.conf.ovs_dpdk.lcore_mask }}
     ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask={{ .Values.conf.ovs_dpdk.lcore_mask | quote }}
+    LCORE_MASK={{ .Values.conf.ovs_dpdk.lcore_mask | quote }}
 {{- end }}
 
     ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:vhost-sock-dir={{ .Values.conf.ovs_dpdk.vhostuser_socket_dir | quote }}
     ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-init=true
+
+  # No need to create the cgroup if lcore_mask or pmd_cpu_mask is not set.
+  if [[ -n ${PMD_CPU_MASK} || -n ${LCORE_MASK} ]]; then
+      # Setup Cgroups to use when breaking out of Kubernetes defined groups
+      mkdir -p /sys/fs/cgroup/cpuset/osh-openvswitch
+      target_mems="/sys/fs/cgroup/cpuset/osh-openvswitch/cpuset.mems"
+      target_cpus="/sys/fs/cgroup/cpuset/osh-openvswitch/cpuset.cpus"
+
+      # Ensure the write target for the for cpuset.mem for the pod exists
+      if [[ -f "$target_mems" && -f "$target_cpus" ]]; then
+        # Write cpuset.mem and cpuset.cpus for new cgroup and add current task to new cgroup
+        cat /sys/fs/cgroup/cpuset/cpuset.mems > "$target_mems"
+        cat /sys/fs/cgroup/cpuset/cpuset.cpus > "$target_cpus"
+        echo $$ > /sys/fs/cgroup/cpuset/osh-openvswitch/tasks
+      else
+        echo "ERROR: Could not find write target for either cpuset.mems: $target_mems or cpuset.cpus: $target_cpus"
+      fi
+  fi
 {{- end }}
 
   exec /usr/sbin/ovs-vswitchd unix:${OVS_SOCKET} \
diff --git a/openvswitch/templates/daemonset-ovs-vswitchd.yaml b/openvswitch/templates/daemonset-ovs-vswitchd.yaml
index 529445a78..5ea8849fe 100644
--- a/openvswitch/templates/daemonset-ovs-vswitchd.yaml
+++ b/openvswitch/templates/daemonset-ovs-vswitchd.yaml
@@ -133,6 +133,8 @@ of hugepages must still be defined in the values.yaml.*/}}
               mountPath: /dev
             - name: pci-drivers
               mountPath: /sys/bus/pci/drivers
+            - name: cgroup
+              mountPath: /sys/fs/cgroup
 {{- end }}
       volumes:
         - name: pod-tmp
@@ -178,5 +180,8 @@ of hugepages must still be defined in the values.yaml.*/}}
           hostPath:
             path: {{ .Values.conf.ovs_dpdk.hugepages_mountpath | quote }}
             type: Directory
+        - name: cgroup
+          hostPath:
+            path: /sys/fs/cgroup
 {{- end }}
 {{- end }}