From f9b03604187645b8b0073ba9716de3f751e23746 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Mon, 23 Oct 2023 17:01:23 +0000 Subject: [PATCH] Mount extra 80Gb volume This PS mounts extra 80Gb volume if available and mounts it to /opt/ext_vol. It also alters docker and containerd configs to move their root folder to that extra volume. This helps zuul gates to succeed when a node with 40Gb volume is assigned to a zuul gate. Change-Id: I1c91b13c233bac5ebfe6e3cb16d4288df2c2fe80 --- playbooks/prepare-hosts.yaml | 1 + roles/deploy-env/defaults/main.yaml | 4 ++ roles/deploy-env/files/containerd_config.toml | 2 +- roles/deploy-env/files/daemon.json | 1 + roles/deploy-env/tasks/containerd.yaml | 2 +- roles/mount-extra-volume/defaults/main.yml | 18 +++++++ roles/mount-extra-volume/tasks/main.yaml | 52 +++++++++++++++++++ zuul.d/jobs.yaml | 8 +++ 8 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 roles/mount-extra-volume/defaults/main.yml create mode 100644 roles/mount-extra-volume/tasks/main.yaml diff --git a/playbooks/prepare-hosts.yaml b/playbooks/prepare-hosts.yaml index c64aa0d65..17ff03ee7 100644 --- a/playbooks/prepare-hosts.yaml +++ b/playbooks/prepare-hosts.yaml @@ -14,4 +14,5 @@ - hosts: all roles: - start-zuul-console + - mount-extra-volume ... diff --git a/roles/deploy-env/defaults/main.yaml b/roles/deploy-env/defaults/main.yaml index 07f340c5b..365e32669 100644 --- a/roles/deploy-env/defaults/main.yaml +++ b/roles/deploy-env/defaults/main.yaml @@ -13,4 +13,8 @@ kubectl: user: zuul group: zuul +docker: + root_path: /var/lib/docker +containerd: + root_path: /var/lib/containerd ... diff --git a/roles/deploy-env/files/containerd_config.toml b/roles/deploy-env/files/containerd_config.toml index 0f2c22e38..4e5902630 100644 --- a/roles/deploy-env/files/containerd_config.toml +++ b/roles/deploy-env/files/containerd_config.toml @@ -3,7 +3,7 @@ imports = [] oom_score = 0 plugin_dir = "" required_plugins = [] -root = "/var/lib/containerd" +root = "{{ containerd.root_path }}" state = "/run/containerd" temp = "" version = 2 diff --git a/roles/deploy-env/files/daemon.json b/roles/deploy-env/files/daemon.json index 254799247..ceb065798 100644 --- a/roles/deploy-env/files/daemon.json +++ b/roles/deploy-env/files/daemon.json @@ -1,4 +1,5 @@ { + "data-root": "{{ docker.root_path }}", "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", "log-opts": { diff --git a/roles/deploy-env/tasks/containerd.yaml b/roles/deploy-env/tasks/containerd.yaml index 833b985c8..372933ec9 100644 --- a/roles/deploy-env/tasks/containerd.yaml +++ b/roles/deploy-env/tasks/containerd.yaml @@ -54,7 +54,7 @@ executable: /bin/bash - name: Configure Docker daemon - copy: + template: src: files/daemon.json dest: /etc/docker/daemon.json diff --git a/roles/mount-extra-volume/defaults/main.yml b/roles/mount-extra-volume/defaults/main.yml new file mode 100644 index 000000000..bdc745576 --- /dev/null +++ b/roles/mount-extra-volume/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +extra_volume: + size: 80G + type: Linux + mount_point: /opt/ext_vol +... diff --git a/roles/mount-extra-volume/tasks/main.yaml b/roles/mount-extra-volume/tasks/main.yaml new file mode 100644 index 000000000..6d6b3348f --- /dev/null +++ b/roles/mount-extra-volume/tasks/main.yaml @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mount additional {{ extra_volume.size }} volume if available + when: + - ansible_distribution == 'Ubuntu' + - (ansible_mounts|selectattr("mount", "equalto", "/")|list)[0].size_available < 50000000000 + block: + - name: Mount additional {{ extra_volume.size }} volume if available + shell: | + set -ex + sudo fdisk --list + df -h + sudo mkdir -p ${EXTRA_VOLUME_MOUNT_POINT} + BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep -E ${EXTRA_VOLUME_SIZE} | grep ${EXTRA_VOLUME_TYPE} | awk '{print $1}') + if ! mount | grep "${BIG_VOLUME}" + then + sudo mkfs.ext4 "${BIG_VOLUME}" + sudo mount "${BIG_VOLUME}" ${EXTRA_VOLUME_MOUNT_POINT} + df -h + fi + environment: + EXTRA_VOLUME_MOUNT_POINT: "{{ extra_volume.mount_point }}" + EXTRA_VOLUME_SIZE: "{{ extra_volume.size }}" + EXTRA_VOLUME_TYPE: "{{ extra_volume.type }}" + +- name: Print configured docker root path + debug: + msg: "Docker root_path: {{ docker.root_path }}" + +- name: Print configured containerd root path + debug: + msg: "containerd root_path: {{ containerd.root_path }}" + +- name: Create mountpoints + shell: | + sudo mkdir -pv "${DOCKER_ROOT_PATH}" + sudo mkdir -pv "${CONTAINERD_ROOT_PATH}" + environment: + DOCKER_ROOT_PATH: "{{ docker.root_path }}" + CONTAINERD_ROOT_PATH: "{{ containerd.root_path }}" +... diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index e35decb0d..00d53720e 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -83,6 +83,14 @@ - playbooks/deploy-env.yaml - playbooks/run-scripts.yaml vars: + extra_volume: + size: 80G + type: Linux + mount_point: /opt/ext_vol + docker: + root_path: "/opt/ext_vol/docker" + containerd: + root_path: "/opt/ext_vol/containerd" # the k8s package versions are available here # https://packages.cloud.google.com/apt/dists/kubernetes-xenial/main/binary-amd64/Packages kube_version: "1.26.3-00"