From 421e59f7bc0f0c077c4790c15be3b053ede0dc14 Mon Sep 17 00:00:00 2001
From: "James E. Blair" <jim@acmegating.com>
Date: Thu, 12 Sep 2024 08:08:03 -0700
Subject: [PATCH] Add debian-bullseye image

This adds an initial debian bullseye dib build job.

Depends-On: https://review.opendev.org/929147
Change-Id: I63f4077305981b7fbce850ae642923cc117a7810
---
 .../opendev-build-diskimage-base/pre.yaml     | 34 ++++++++
 .../make-source-repositories-cache/README.rst | 12 +++
 .../defaults/main.yaml                        |  2 +
 .../tasks/main.yaml                           | 27 ++++++
 zuul.d/jobs.yaml                              | 87 +++++++++++++++++++
 zuul.d/project.yaml                           |  4 +-
 6 files changed, 164 insertions(+), 2 deletions(-)
 create mode 100644 playbooks/opendev-build-diskimage-base/pre.yaml
 create mode 100644 roles/make-source-repositories-cache/README.rst
 create mode 100644 roles/make-source-repositories-cache/defaults/main.yaml
 create mode 100644 roles/make-source-repositories-cache/tasks/main.yaml
 create mode 100644 zuul.d/jobs.yaml

diff --git a/playbooks/opendev-build-diskimage-base/pre.yaml b/playbooks/opendev-build-diskimage-base/pre.yaml
new file mode 100644
index 0000000..e0c11a7
--- /dev/null
+++ b/playbooks/opendev-build-diskimage-base/pre.yaml
@@ -0,0 +1,34 @@
+- hosts: all
+  pre_tasks:
+    - name: Install custom element requirements
+      become: true
+      package:
+        state: present
+        name:
+          - python3-yaml
+    - name: Setup dib directories
+      become: true
+      shell: |
+        set -x
+        
+        mkdir -p /opt/dib_cache
+        chown {{ ansible_user }} /opt/dib_cache
+        chgrp {{ ansible_user }} /opt/dib_cache
+
+        mkdir -p /opt/dib_tmp
+
+        if [ -b /dev/vdb ]; then
+          DEV='/dev/vdb'
+        elif [ -b /dev/xvde ]; then
+          DEV='/dev/xvde'
+        fi
+
+        if [ -n "$DEV" ]; then
+          mkfs.ext4 $DEV
+          mount $DEV /opt/dib_tmp
+        fi
+
+        chown {{ ansible_user }} /opt/dib_tmp
+        chgrp {{ ansible_user }} /opt/dib_tmp
+  roles:
+    - make-source-repositories-cache
diff --git a/roles/make-source-repositories-cache/README.rst b/roles/make-source-repositories-cache/README.rst
new file mode 100644
index 0000000..73e3378
--- /dev/null
+++ b/roles/make-source-repositories-cache/README.rst
@@ -0,0 +1,12 @@
+Reformat the git repo cache for source-respositories
+
+This reverses the process of creating the /opt/git cache on worker
+nodes.  It turns that cache into the format used by the
+source-repositories diskimage builder element.
+
+**Role Variables**
+
+.. zuul:rolevar:: cached_repos_root
+   :default: /opt/git
+
+   The root of the cached repos.
diff --git a/roles/make-source-repositories-cache/defaults/main.yaml b/roles/make-source-repositories-cache/defaults/main.yaml
new file mode 100644
index 0000000..4c8a06d
--- /dev/null
+++ b/roles/make-source-repositories-cache/defaults/main.yaml
@@ -0,0 +1,2 @@
+# Matches prepare_workspace_root
+cached_repos_root: /opt/git
diff --git a/roles/make-source-repositories-cache/tasks/main.yaml b/roles/make-source-repositories-cache/tasks/main.yaml
new file mode 100644
index 0000000..4bf265f
--- /dev/null
+++ b/roles/make-source-repositories-cache/tasks/main.yaml
@@ -0,0 +1,27 @@
+- name: Reformat the git repo cache for source-repositories
+  environment: "{{ build_diskimage_environment }}"
+  args:
+    executable: /bin/bash
+  shell: |
+    GIT_CACHE_ROOT="{{ cached_repos_root }}"
+
+    REPOTYPE=git
+    CACHE_BASE=${DIB_IMAGE_CACHE}/source-repositories
+    mkdir -p ${CACHE_BASE}
+
+    # Note: maxdepth 4 is based on the current opendev practice of
+    # not having repos deeper than two directory levels.  This is
+    # fragile and should probably be replaced with something better,
+    # but as written it saves considerable time.
+    for NODE_CACHE_PATH in $(find $GIT_CACHE_ROOT -maxdepth 4 -type d -name .git); do
+        REPOPATH=$(dirname $NODE_CACHE_PATH)
+        REPONAME=$(basename $REPOPATH)
+        REPOLOCATION="$(echo $REPOPATH | sed "s,$GIT_CACHE_ROOT/,https://,").git"
+
+        # From diskimage-builder element source-repositories
+        CACHE_NAME=$(echo "${REPOTYPE}_${REPOLOCATION}" | sha1sum | awk '{ print $1 }' )
+        CACHE_NAME=${REPONAME//[^A-Za-z0-9]/_}_${CACHE_NAME}
+        CACHE_PATH=${CACHE_BASE}/${CACHE_NAME}
+
+        mv ${REPOPATH} ${CACHE_PATH}
+    done
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
new file mode 100644
index 0000000..5028e6f
--- /dev/null
+++ b/zuul.d/jobs.yaml
@@ -0,0 +1,87 @@
+- job:
+    name: opendev-build-diskimage-base
+    description: |
+      The abstract base job for building VM images used in OpenDev's Zuul
+    parent: build-diskimage
+    timeout: 7200
+    abstract: true
+    pre-run:
+      - playbooks/opendev-build-diskimage-base/pre.yaml
+    vars:
+      # TODO(corvus): This will eventually be supplied by Zuul
+      build_diskimage_formats:
+        - qcow2
+      build_diskimage_elements: &base_elements
+        - vm
+        - simple-init
+        - openstack-repos
+        - nodepool-base
+        - growroot
+        - infra-package-needs
+      build_diskimage_environment:
+        TMPDIR: /opt/dib_tmp
+        ELEMENTS_PATH: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/opendev/zuul-jobs'].src_dir }}/dib-elements"
+        # TODO: Do we want streaming logs?
+        DIB_QUIET: '0'
+        DIB_NO_TMPFS: '1'
+        DIB_CHECKSUM: '1'
+        DIB_IMAGE_CACHE: /opt/dib_cache
+        DIB_JOURNAL_SIZE: '512'
+        DIB_GRUB_TIMEOUT: '0'
+        GIT_HTTP_LOW_SPEED_TIME: '300'
+        GIT_HTTP_LOW_SPEED_LIMIT: '1000'
+        DIB_SHOW_IMAGE_USAGE: '1'
+        ZUUL_USER_SSH_PUBLIC_KEY: |
+          ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDh5u0DWNi0d3uqI82izAxVTUTrGl36L3szEqV9WrilGmxaTtp9X7HrktJ5J+rvxQqz39llTf1v7iYA4CNKto/48RBAB0mKEEI4x4iw+
+          fu/BLU7nu7ewSfXLUxHahxaTgIk2KcbegknD5NzMjalyfNfgTPDGv8BjwHeeNdZmJMBvPFGu6fO48M4yK1tiQn0kAkaH4oII/M4pyF8vy9tPTODAD7RvnMvQAb08LZZvE/IPzJAHNXFRb1v+DBa38
+          fOvdyaz/nibrsxiOWZxQVLgjYciUeDy1xvXADaWlqvxmLy+90LHbJFbGxK4AN0mWfwBiUMVyxZjkun39pjTNl2k09OhOq+R52UqnehMc4eBdZCddnCUq4/efbFCJkqe5wY+SE8fYybJjauUL64zyr
+          wf6yfWkXvPVHWa9Y+NCmvH8PCBUcsQnwO7l/Yb4N+8+u6zkODyuc9wLAY+DpnptE3plXtvUs5negC4fvJSnOHpWXuoi9yzp7IlPf6fSjMMDQo0JjCYJwazdzqrIH2VSCcfHAqWF0ECR8IgwZV1bp0
+          xFe0UN0Gjsgkozqf8rvs1AYyTSeD19Wg9j+crTke8E1sfoI/qFzHwzBQFKJ+2l0cs7pZWJBARlhbt1j1IouS2aH+74xwsavRhBz4IsFTPqWiP6JTrgk5cgKRnTqInzNfdaLeUw== zuul-worker@
+          openstack.org
+
+- job:
+    name: opendev-build-diskimage-base-debuntu
+    description: |
+      An abstract base job for building Debian or Ubuntu based VM images
+    parent: opendev-build-diskimage-base
+    abstract: true
+    vars:
+      build_diskimage_environment:
+        DIB_APT_LOCAL_CACHE: '0'
+        DIB_DISABLE_APT_CLEANUP: '1'
+        DIB_DEBOOTSTRAP_EXTRA_ARGS: '--no-check-gpg'
+
+- job:
+    name: opendev-build-diskimage-debian-bullseye
+    description: |
+      Build the OpenDev Debian bullseye VM image
+    parent: opendev-build-diskimage-base-debuntu
+    vars:
+      build_diskimage_image_name: debian-bullseye
+      build_diskimage_elements:
+        - *base_elements
+        - debian-minimal
+        - cache-devstack
+      build_diskimage_environment:
+        DIB_RELEASE: 'bullseye'
+        DIB_DISTRIBUTION_MIRROR: 'https://{{ zuul_site_mirror_fqdn }}/debian'
+        DIB_DEBIAN_SECURITY_MIRROR:
+          'https://{{ zuul_site_mirror_fqdn }}/debian-security'
+        DIB_BLOCK_DEVICE_CONFIG: |
+          # Default single partition loopback
+          - local_loop:
+              name: image0
+  
+          - partitioning:
+              base: image0
+              label: mbr
+              partitions:
+                - name: root
+                  flags: [ boot, primary ]
+                  size: 100%
+                  mkfs:
+                    mount:
+                      mount_point: /
+                      fstab:
+                        options: "defaults,nobarrier,noatime"
+                        fsck-passno: 1
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index e30e3ff..816bf9b 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -1,7 +1,7 @@
 - project:
     check:
       jobs:
-        - noop
+        - opendev-build-diskimage-debian-bullseye
     gate:
       jobs:
-        - noop
+        - opendev-build-diskimage-debian-bullseye