Move nodepool functests to podman

Now that nodepool images are on quay.io we don't get speculative
container image testing with docker. The reason for this is docker only
knows how to lookup images hosted by docker.io in mirrors which
specualtive container image testing relies on. Since the images are
hosted on quay.io instead of docker.io we lose this functionality.

Address this by switching to podman and podman-compose which does
understand how to fetch images with mirrors from any location.

Depends-On: https://review.opendev.org/c/zuul/zuul/+/687135
Change-Id: I1a510a9b68a2f01098f3c099a129d6d268b422d9

playbooks/nodepool-functional-container-openstack/post.yaml

@@ -1,7 +1,10 @@
 - hosts: all
-  roles:
-    - collect-container-logs
   tasks:
+    - name: Collect container logs
+      include_role:
+        name: collect-container-logs
+      vars:
+        container_command: podman
     - name: Copy nodepool logs
       ignore_errors: yes
       block:
@@ -32,9 +35,9 @@
             dest: '{{ zuul.executor.log_root }}'
             mode: pull
 
-        - name: Copy docker logs
+        - name: Copy podman logs
           become: True
           synchronize:
-            src: '{{ ansible_user_dir }}/zuul-output/logs/docker'
+            src: '{{ ansible_user_dir }}/zuul-output/logs/podman'
             dest: '{{ zuul.executor.log_root }}'
             mode: pull

playbooks/nodepool-functional-container-openstack/pre.yaml

@@ -7,15 +7,15 @@
       zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"
     - role: ensure-zookeeper
       zookeeper_use_tls: true
-    - ensure-docker
-    # Note: keep after ensure-docker
+    - ensure-podman
+    # Note: keep after ensure-podman
     - use-buildset-registry
     - ensure-devstack
   tasks:
-    - name: Install docker-compose dependencies
-      package:
+    - name: Install podman-compose dependencies
+      pip:
         name:
-          - docker-compose
+          - podman-compose
       become: yes
     - name: Create a local user that matches the container UID
       user:

playbooks/nodepool-functional-container-openstack/run.yaml

@@ -86,25 +86,34 @@
       loop:
         - '{{ nodepool_log_dir }}'
 
-    - name: Write docker-compose
+    - name: Write docker-compose.yaml
       template:
         src: docker-compose.yaml.j2
         dest: /etc/nodepool/docker-compose.yaml
         mode: 0600
 
-    - name: Run docker compose pull
+    - name: Run podman compose pull
       shell:
-        cmd: docker-compose pull
+        cmd: podman-compose pull
         chdir: /etc/nodepool
+      # We run as root to allow us to move nodepool processes into a new
+      # process cgroup which enables podman to run nested in docker.
+      become: yes
 
-    - name: Run docker compose up
+    - name: Run podman compose up
       shell:
-        cmd: docker-compose up -d --timeout 60
+        cmd: podman-compose up -d --timeout 60
         chdir: /etc/nodepool
+      # We run as root to allow us to move nodepool processes into a new
+      # process cgroup which enables podman to run nested in docker.
+      become: yes
 
     - name: Cleanup unused images
       shell:
-        cmd: docker image prune -f
+        cmd: podman image prune -f
+      # We run as root to allow us to move nodepool processes into a new
+      # process cgroup which enables podman to run nested in docker.
+      become: yes
 
     - name: Check nodepool functionality
       command: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}/tools/functional-test-check.sh"

playbooks/nodepool-functional-container-openstack/templates/docker-compose.yaml.j2

@@ -5,7 +5,7 @@ volumes:
 
 services:
   nodepool-builder:
-    image: zuul/nodepool-builder{{ nodepool_container_tag|default('') }}
+    image: quay.io/zuul-ci/nodepool-builder{{ nodepool_container_tag|default('') }}
     network_mode: host
     restart: always
 
@@ -33,7 +33,7 @@ services:
       - /opt/zookeeper/ca:/opt/zookeeper/ca:ro
 
   nodepool-launcher:
-    image: zuul/nodepool-launcher{{ nodepool_container_tag|default('') }}
+    image: quay.io/zuul-ci/nodepool-launcher{{ nodepool_container_tag|default('') }}
     network_mode: host
     restart: always
 

tools/functional-test-check.sh

@@ -11,7 +11,7 @@ if [[ ${NODEPOOL_FUNCTIONAL_CHECK:-} == "installed" ]]; then
     NODEPOOL_CONFIG=${NODEPOOL_CONFIG:-/etc/nodepool/nodepool.yaml}
     NODEPOOL="$NODEPOOL_INSTALL/bin/nodepool -c $NODEPOOL_CONFIG"
 elif [[ ${NODEPOOL_FUNCTIONAL_CHECK:-} == "containers" ]]; then
-    NODEPOOL="docker exec nodepool_nodepool-launcher_1 nodepool"
+    NODEPOOL="sudo podman exec nodepool_nodepool-launcher_1 nodepool"
 else
     echo "Running in unknown environment!"
     exit 1