Test buildset registry with k8s and docker

This adds a test job for the buildset registry with k8s with the
docker runtime.

Change-Id: I0ec1d3d5980424f85f5e7619b9dec96c8f0f70f1

roles/run-buildset-registry/templates/registry.yaml.j2

@@ -1,7 +1,7 @@
 registry:
   address: '::'
   port: 5000
-  public-url: 'https://{{ ansible_host | ipwrap }}:5000'
+  public-url: 'https://{{ ansible_host | ipwrap }}:{{ buildset_registry_port }}'
   tls-cert: /tls/cert.pem
   tls-key: /tls/cert.key
   secret: "{{ registry_secret }}"

test-playbooks/registry/buildset-registry-k8s-docker-post.yaml

@@ -0,0 +1,8 @@
+- hosts: all
+  tasks:
+    - name: Describe dockertest pod
+      command: kubectl describe pod/dockertest
+      ignore_errors: true
+    - name: Describe upstream-dockertest pod
+      command: kubectl describe pod/upstream-dockertest
+      ignore_errors: true

test-playbooks/registry/buildset-registry-k8s-docker.yaml

@@ -0,0 +1,26 @@
+- hosts: all
+  roles:
+    - role: clear-firewall
+    - role: install-kubernetes
+    - role: use-buildset-registry
+      buildset_registry_docker_user: root
+  tasks:
+    - name: Wait for cluster to come up
+      command: kubectl cluster-info
+      register: result
+      until: result.rc == 0
+      retries: 5
+      delay: 30
+    - name: Run a local test pod
+      command: kubectl run --generator=run-pod/v1 --image=zuul/docker-testimage dockertest
+    - name: Wait for the pod to be ready
+      command: kubectl wait --for=condition=Ready pod/dockertest --timeout=60s
+    - name: Check the output of the pod
+      shell: "kubectl logs pod/dockertest | grep 'Zuul container test'"
+
+    - name: Run a remote test pod
+      command: kubectl run --generator=run-pod/v1 --image=debian:testing upstream-dockertest --command -- /bin/bash -c 'echo Upstream; sleep infinity'
+    - name: Wait for the pod to be ready
+      command: kubectl wait --for=condition=Ready pod/upstream-dockertest --timeout=60s
+    - name: Check the output of the pod
+      shell: "kubectl logs pod/upstream-dockertest | grep 'Upstream'"

test-playbooks/registry/buildset-registry-pre.yaml

@@ -0,0 +1,39 @@
+- hosts: all
+  tasks:
+    - name: Add project atomic PPA
+      become: true
+      apt_repository:
+        repo: ppa:projectatomic/ppa
+    - name: Install packages
+      become: true
+      package:
+        name:
+          - skopeo
+        state: present
+
+    # This happens in the pre-playbook of the
+    # opendev-buildset-registry job.  Since we're testing these roles
+    # in opendev, and we're running a second buildset registry, we
+    # need to do the same here for it.
+    - name: Open the IPv4 port for the second buildset registry
+      become: true
+      iptables:
+        action: insert
+        chain: openstack-INPUT
+        destination_port: '9000'
+        jump: ACCEPT
+        match: tcp
+        ctstate: NEW
+        protocol: tcp
+        ip_version: ipv4
+    - name: Open the IPv6 port for the second buildset registry
+      become: true
+      iptables:
+        action: insert
+        chain: openstack-INPUT
+        destination_port: '9000'
+        jump: ACCEPT
+        match: tcp
+        ctstate: NEW
+        protocol: tcp
+        ip_version: ipv6

test-playbooks/registry/buildset-registry.yaml

@@ -0,0 +1,32 @@
+# This job inherits from a buildset-registry job, so it should already
+# be running locally and have any speculative images loaded into it.
+- hosts: all
+  tasks:
+    - name: Load real buildset registry connection info
+      set_fact:
+        real_buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
+    # This should now use the speculative image, because we've already
+    # run use-buildset-registry.
+    - name: Run the fake buildset registry
+      include_role:
+        role: run-buildset-registry
+      vars:
+        buildset_registry_root: "{{ ansible_user_dir }}/fake_buildset_registry"
+        buildset_registry_port: 9000
+    # Leave that zuul return so that dependent jobs use the fake one
+    - name: Load fake buildset registry connection info
+      set_fact:
+        fake_buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
+    - name: Build a test image
+      command: "docker build . -t zuul/testimage:latest"
+      args:
+        chdir: "{{ zuul.project.src_dir }}/test-playbooks/registry/docker"
+    - name: Push test image into fake buildset registry
+      command: "skopeo copy --dest-tls-verify=false --dest-creds {{ buildset_registry.username }}:{{ buildset_registry.password }} docker-daemon:zuul/testimage:latest docker://localhost:9000/zuul/docker-testimage:latest"
+    - name: Push test image into fake buildset registry
+      command: "skopeo copy --dest-tls-verify=false --dest-creds {{ buildset_registry.username }}:{{ buildset_registry.password }} docker-daemon:zuul/testimage:latest docker://localhost:9000/quay.io/zuul/quay-testimage:latest"
+    - name: Pause the job
+      zuul_return:
+        data:
+          zuul:
+            pause: true

test-playbooks/registry/docker/Dockerfile

@@ -1,2 +1,2 @@
 FROM debian:testing
-RUN touch /tmp/foo
+CMD echo "Zuul container test"; sleep infinity

test-playbooks/registry/test-registry-post.yaml

@@ -15,6 +15,7 @@
       shell: "docker logs {{ item }} &> {{ ansible_user_dir }}/zuul-output/logs/docker/{{ item }}.txt"
       args:
         executable: /bin/bash
+      ignore_errors: true
 
     - name: Open container logs permissions
       file:

zuul-tests.d/container-roles-jobs.yaml

@@ -26,6 +26,87 @@
         - name: builder
           label: ubuntu-bionic
 
+- job:
+    name: zuul-jobs-test-registry-buildset-registry
+    parent: opendev-buildset-registry
+    description: |
+      Run a buildset registry for the test-registry jobs
+
+      This runs two registries: a real buildset registry so that we
+      can receive speculative zuul-registry images, and a fake
+      buildset registry (running the speculative or latest
+      zuul-registry) that is used to test using the buildset registry
+      role.
+
+      It is not meant to be used directly but rather run on changes
+      to roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/install-docker/.*
+      - roles/install-kubernetes/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    pre-run: test-playbooks/registry/buildset-registry-pre.yaml
+    run: test-playbooks/registry/buildset-registry.yaml
+    post-run: test-playbooks/registry/test-registry-post.yaml
+
+- job:
+    name: zuul-jobs-test-registry-buildset-registry-k8s-docker
+    dependencies: zuul-jobs-test-registry-buildset-registry
+    description: |
+      Test a buildset registry with kubernetes and docker
+
+      It is not meant to be used directly but rather run on changes
+      to roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/install-docker/.*
+      - roles/install-kubernetes/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    run: test-playbooks/registry/buildset-registry-k8s-docker.yaml
+    post-run:
+      - test-playbooks/registry/buildset-registry-k8s-docker-post.yaml
+      - test-playbooks/registry/test-registry-post.yaml
+
+- job:
+    name: zuul-jobs-test-install-kubernetes-docker
+    description: |
+      Test the install-kubernetes role with docker
+
+      This job tests changes to the install-kubernetes roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    files:
+      - roles/install-docker/.*
+      - roles/install-kubernetes/.*
+      - test-playbooks/install-kubernetes.yaml
+    run: test-playbooks/install-kubernetes/docker.yaml
+    post-run: test-playbooks/install-kubernetes/post.yaml
+
+- job:
+    name: zuul-jobs-test-install-kubernetes-crio
+    description: |
+      Test the install-kubernetes role with crio-o
+
+      This job tests changes to the install-kubernetes roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-bionic
+        - name: executor
+          label: ubuntu-bionic
+        - name: builder
+          label: ubuntu-bionic
+
 - job:
     name: zuul-jobs-test-install-kubernetes-docker
     description: |
@@ -62,6 +143,8 @@
     check:
       jobs: &id001
         - zuul-jobs-test-registry
+        - zuul-jobs-test-registry-buildset-registry
+        - zuul-jobs-test-registry-buildset-registry-k8s-docker
         - zuul-jobs-test-install-kubernetes-docker
         - zuul-jobs-test-install-kubernetes-crio
     gate: