From b5148387ba3abb84dd11e862e74d737389092406 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 20 Nov 2024 09:06:02 -0800
Subject: [PATCH] Upgrade and reboot test nodes before openafs installation

CentOS ARM in particular complains bitterly if we try to upgrade openafs
on it and the kernel it is running is stale compared to packges in the
mirrors. This happens if our images have lagged behind upstream package
updates. To mitigate this we upgrade and reboot the system prior to
testing openafs installation.

To make arm64 reboots onto the new kernel reliable we manually run
grubby --set-default to explicitly set the default kernel to boot to the
newly installed kernel. Then we remove the old kernel with grubby and
finally generate a new grub config. This is necessary likely due to:

  https://bugzilla.redhat.com/show_bug.cgi?id=2032680

Note we also do the reboot (but not grub(by) dance) on Debuntu for
symmetry.

Change-Id: If87a0b1d7dc063ac9122d85f65b6fe1c129d2052
---
 roles-test/pre.yaml             | 59 +++++++++++++++++++++++++++++++++
 zuul.d/system-config-roles.yaml |  1 +
 2 files changed, 60 insertions(+)
 create mode 100644 roles-test/pre.yaml

diff --git a/roles-test/pre.yaml b/roles-test/pre.yaml
new file mode 100644
index 0000000000..4dca3e7c3e
--- /dev/null
+++ b/roles-test/pre.yaml
@@ -0,0 +1,59 @@
+# Do these updates particularly for CentOS arm which may have stale kernels
+# preventing openafs installation
+- name: Update and reboot nodes before installing openafs
+  hosts: all
+  tasks:
+    - name: Update CentOS nodes
+      when: ansible_distribution == "CentOS"
+      block:
+        - name: DNF Update
+          dnf:
+            name: "*"
+            state: latest
+          become: yes
+
+        - name: Hacky script to force default kernel to new version
+          shell: |
+            set -x
+            # Get the newest kernel version in /boot
+            NEWEST=$(ls /boot | grep vmlinuz | sort -V -r | head -1)
+            OLDEST=$(ls /boot | grep vmlinuz | sort -V | head -1)
+            grubby --set-default=/boot/$NEWEST
+            if [[ "$OLDEST" != "$NEWEST" ]] ; then
+              grubby --remove-kernel=/boot/$OLDEST
+            fi
+          args:
+            executable: /usr/bin/bash
+          become: yes
+
+        - name: Tell grub about the new kernel setup
+          command: grub2-mkconfig --update-bls-cmdline -o /boot/grub2/grub.cfg
+          become: yes
+
+    - name: Update Debuntu nodes
+      when: ansible_distribution == "Ubuntu" or ansible_distribution == "Debian"
+      block:
+        - name: Apt upgrade
+          apt:
+            name: "*"
+            state: latest
+          become: yes
+
+    - name: Record running kernel version
+      command: uname -a
+
+    - name: Reboot
+      reboot:
+        reboot_timeout: 900
+      become: yes
+
+    - name: Restart zuul console log daemon
+      include_role:
+        name: start-zuul-console
+
+    - name: Record running kernel version
+      command: uname -a
+
+    - name: Pause for a bit to ensure system is up post reboot
+      pause:
+        seconds: 60
diff --git a/zuul.d/system-config-roles.yaml b/zuul.d/system-config-roles.yaml
index 0ab8648668..aa41a2d2c5 100644
--- a/zuul.d/system-config-roles.yaml
+++ b/zuul.d/system-config-roles.yaml
@@ -9,6 +9,7 @@
     description: Test roles provided by system-config with Zuul.
     abstract: true
     parent: base
+    pre-run: roles-test/pre.yaml
     run: roles-test/base.yaml
     post-run: roles-test/post.yaml
     files: