Disable libvirt cgroup functionality for cgroup v2

Change-Id: I5a9f1828d7c2f36e14de89323868c4a1dbebba64

libvirt/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm libvirt
 name: libvirt
-version: 0.1.20
+version: 0.1.21
 home: https://libvirt.org
 sources:
   - https://libvirt.org/git/?p=libvirt.git;a=summary

libvirt/templates/bin/_libvirt.sh.tpl

@@ -16,6 +16,13 @@ limitations under the License.
 
 set -ex
 
+# TODO: We disable cgroup functionality for cgroup v2, we should fix this in the future
+if $(stat -fc %T /sys/fs/cgroup/ | grep -q cgroup2fs); then
+  CGROUP_VERSION=v2
+else
+  CGROUP_VERSION=v1
+fi
+
 if [ -n "$(cat /proc/*/comm 2>/dev/null | grep -w libvirtd)" ]; then
   set +x
   for proc in $(ls /proc/*/comm 2>/dev/null); do
@@ -38,14 +45,16 @@ if [[ -c /dev/kvm ]]; then
     chown root:kvm /dev/kvm
 fi
 
-#Setup Cgroups to use when breaking out of Kubernetes defined groups
+if [ $CGROUP_VERSION != "v2" ]; then
-CGROUPS=""
+  #Setup Cgroups to use when breaking out of Kubernetes defined groups
-for CGROUP in cpu rdma hugetlb; do
+  CGROUPS=""
-  if [ -d /sys/fs/cgroup/${CGROUP} ]; then
+  for CGROUP in cpu rdma hugetlb; do
-    CGROUPS+="${CGROUP},"
+    if [ -d /sys/fs/cgroup/${CGROUP} ]; then
-  fi
+      CGROUPS+="${CGROUP},"
-done
+    fi
-cgcreate -g ${CGROUPS%,}:/osh-libvirt
+  done
+  cgcreate -g ${CGROUPS%,}:/osh-libvirt
+fi
 
 # We assume that if hugepage count > 0, then hugepages should be exposed to libvirt/qemu
 hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')"
@@ -68,43 +77,49 @@ if [ 0"$hp_count" -gt 0 ]; then
     exit 1
   fi
 
-  # Kubernetes 1.10.x introduced cgroup changes that caused the container's
+  if [ $CGROUP_VERSION != "v2" ]; then
-  # hugepage byte limit quota to zero out. This workaround sets that pod limit
+    # Kubernetes 1.10.x introduced cgroup changes that caused the container's
-  # back to the total number of hugepage bytes available to the baremetal host.
+    # hugepage byte limit quota to zero out. This workaround sets that pod limit
-  if [ -d /sys/fs/cgroup/hugetlb ]; then
+    # back to the total number of hugepage bytes available to the baremetal host.
-    limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \
+    if [ -d /sys/fs/cgroup/hugetlb ]; then
-      (echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?"
+      limits="$(ls /sys/fs/cgroup/hugetlb/{{ .Values.conf.kubernetes.cgroup }}/hugetlb.*.limit_in_bytes)" || \
-       exit 1)
+        (echo "ERROR: Failed to locate any hugetable limits. Did you set the correct cgroup in your values used for this chart?"
-    for limit in $limits; do
+         exit 1)
-      target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)"
+      for limit in $limits; do
-      # Ensure the write target for the hugepage limit for the pod exists
+        target="/sys/fs/cgroup/hugetlb/$(dirname $(awk -F: '($2~/hugetlb/){print $3}' /proc/self/cgroup))/$(basename $limit)"
-      if [ ! -f "$target" ]; then
+        # Ensure the write target for the hugepage limit for the pod exists
-        echo "ERROR: Could not find write target for hugepage limit: $target"
+        if [ ! -f "$target" ]; then
-      fi
+          echo "ERROR: Could not find write target for hugepage limit: $target"
+        fi
 
-      # Write hugetable limit for pod
+        # Write hugetable limit for pod
-      echo "$(cat $limit)" > "$target"
+        echo "$(cat $limit)" > "$target"
-    done
+      done
-  fi
+    fi
 
-  # Determine OS default hugepage size to use for the hugepage write test
+    # Determine OS default hugepage size to use for the hugepage write test
-  default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')"
+    default_hp_kb="$(cat /proc/meminfo | grep Hugepagesize | tr -cd '[:digit:]')"
 
-  # Attempt to write to the hugepage mount to ensure it is operational, but only
+    # Attempt to write to the hugepage mount to ensure it is operational, but only
-  # if we have at least 1 free page.
+    # if we have at least 1 free page.
-  num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')"
+    num_free_pages="$(cat /sys/kernel/mm/hugepages/hugepages-${default_hp_kb}kB/free_hugepages | tr -cd '[:digit:]')"
-  echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB"
+    echo "INFO: '$num_free_pages' free hugepages of size ${default_hp_kb}kB"
-  if [ 0"$num_free_pages" -gt 0 ]; then
+    if [ 0"$num_free_pages" -gt 0 ]; then
-    (fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \
+      (fallocate -o0 -l "$default_hp_kb" /dev/hugepages/foo && rm /dev/hugepages/foo) || \
-      (echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB"
+        (echo "ERROR: fallocate failed test at /dev/hugepages with size ${default_hp_kb}kB"
-       rm /dev/hugepages/foo
+         rm /dev/hugepages/foo
-       exit 1)
+         exit 1)
+    fi
   fi
 fi
 
 if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] || [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
-  #NOTE(portdirect): run libvirtd as a transient unit on the host with the osh-libvirt cgroups applied.
+  if [ $CGROUP_VERSION != "v2" ]; then
-  cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen &
+    #NOTE(portdirect): run libvirtd as a transient unit on the host with the osh-libvirt cgroups applied.
+    cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen &
+  else
+    systemd-run --scope --slice=system libvirtd --listen &
+  fi
 
   tmpsecret=$(mktemp --suffix .xml)
   if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then
@@ -180,5 +195,9 @@ EOF
 
 fi
 
-#NOTE(portdirect): run libvirtd as a transient unit on the host with the osh-libvirt cgroups applied.
+if [ $CGROUP_VERSION != "v2" ]; then
-cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen
+  #NOTE(portdirect): run libvirtd as a transient unit on the host with the osh-libvirt cgroups applied.
+  cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen
+else
+  systemd-run --scope --slice=system libvirtd --listen
+fi

releasenotes/notes/libvirt.yaml

@@ -21,4 +21,5 @@ libvirt:
   - 0.1.18 Replace node-role.kubernetes.io/master with control-plane
   - 0.1.19 Set kubernetes cgroup value equal kubepods.slice to fit systemd cgroup driver
   - 0.1.20 Update Ceph to 17.2.6
+  - 0.1.21 Disable libvirt cgroup functionality for cgroup-v2
 ...