Merge "Add mirror-container-images role and job"

.ansible-lint

@@ -12,6 +12,7 @@ skip_list:
   - key-order  # This suggests an arbitrary ordering of keys which means our developers would need to memorize someone's personal preference
   - name[play]  # We have no objection to naming plays, but because of all of our short job playbooks, there are a *lot* of them that are pretty self-evident in the Zuul context, so it doesn't seem necessary.
   - name[casing]  # We should use proper capitalization, but this is too nit-picky to waste time on
+  - name[template]  # This is too simple of a check and does not allow for templates like "foo {{ bar }}-{{ baz }}", and is also fairly "personal preference"
 
   # NOTE(ianw) After following the jinja formatting extensions in
   # ansible-lint for a few releases, this does not seem to be

doc/source/container-jobs.rst

@@ -4,3 +4,4 @@ Container Jobs
 .. zuul:autojob:: build-container-image
 .. zuul:autojob:: upload-container-image
 .. zuul:autojob:: promote-container-image
+.. zuul:autojob:: mirror-container-images

doc/source/container-roles.rst

@@ -11,6 +11,7 @@ Container Roles
 .. zuul:autorole:: ensure-podman
 .. zuul:autorole:: ensure-skopeo
 .. zuul:autorole:: ensure-quay-repo
+.. zuul:autorole:: mirror-container-images
 .. zuul:autorole:: pause-buildset-registry
 .. zuul:autorole:: promote-container-image
 .. zuul:autorole:: promote-docker-image

playbooks/container-image/README.rst

@@ -6,6 +6,10 @@ context:
   * :zuul:job:`upload-container-image`: Build and stage the images in a registry.
   * :zuul:job:`promote-container-image`: Promote previously uploaded images.
 
+The following utility job may also be useful:
+
+  * :zuul:job:`mirror-container-images`: Copy existing images from one registry to another.
+
 The jobs can work in multiple modes depending on your requirements.
 They all accept the same input data, principally a list of
 dictionaries representing the images to build.  YAML anchors_ can be

playbooks/container-image/mirror.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - mirror-container-images

roles/mirror-container-images/README.rst

@@ -0,0 +1,72 @@
+Copy container images from one registry to another.
+
+.. zuul:rolevar:: mirror_container_images_images
+   :type: list
+
+   A list of container images to copy.
+
+   .. zuul:rolevar:: src_repository
+      :type: str
+
+      The source image repository (including registry name).
+
+   .. zuul:rolevar:: src_tag
+      :type: str
+
+      The source image tag.
+
+   .. zuul:rolevar:: dest_repository
+      :type: str
+
+      The destination image repository (including registry name).
+
+   .. zuul:rolevar:: dest_tag
+      :type: str
+
+      The destination image tag.
+
+   .. zuul:rolevar:: dest_registry
+      :type: str
+
+      The name of the registry to which the destination image will be
+      pushed.
+
+.. zuul:rolevar:: container_registry_credentials
+   :type: dict
+
+   This is expected to be a Zuul Secret in dictionary form.  Each key
+   is the name of a registry, and its value a dictionary with
+   information about that registry.
+
+   Example:
+
+   .. code-block:: yaml
+
+      container_registry_credentials:
+        quay.io:
+          username: foo
+          password: bar
+
+   .. zuul:rolevar:: [registry name]
+      :type: dict
+
+      Information about a registry.  The key is the registry name, and
+      its value a dict as follows:
+
+      .. zuul:rolevar:: username
+
+         The registry username.
+
+      .. zuul:rolevar:: password
+
+         The registry password.
+
+      .. zuul:rolevar:: repository
+
+         Optional; if supplied this is a regular expression which
+         restricts to what repositories the image may be uploaded.  The
+         following example allows projects to upload images to
+         repositories within an organization based on their own names::
+
+           repository: "^myorgname/{{ zuul.project.short_name }}.*"
+

roles/mirror-container-images/tasks/inner.yaml

@@ -0,0 +1,31 @@
+- name: Verify repository names
+  when: |
+    container_registry_credentials is defined
+    and zj_image.dest_registry not in container_registry_credentials
+  fail:
+    msg: "{{ zj_image.dest_registry }} credentials not found"
+
+- name: Verify repository permission
+  when: |
+    container_registry_credentials[zj_image.dest_registry].repository is defined and
+    not zj_image.dest_repository | regex_search(container_registry_credentials[zj_image.dest_registry].repository)
+  fail:
+    msg: "{{ zj_image.repository }} not permitted by {{ container_registry_credentials[zj_image.dest_registry].repository }}"
+
+- name: Log in to registry
+  command: "{{ container_command }} login -u {{ container_registry_credentials[zj_image.dest_registry].username }} -p {{ container_registry_credentials[zj_image.dest_registry].password }} {{ zj_image.dest_registry }}"
+  no_log: true
+
+- name: Push and pull image
+  block:
+    - name: "Pull image {{ zj_image.src_repository }}:{{ zj_image.src_tag }}"
+      command: "{{ container_command }} pull {{ zj_image.src_repository }}:{{ zj_image.src_tag }}"
+
+    - name: Retag image
+      command: "{{ container_command }} tag {{ zj_image.src_repository }}:{{ zj_image.src_tag }} {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}"
+
+    - name: "Push image {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}"
+      command: "{{ container_command }} push {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}"
+  always:
+    - name: Log out of registry
+      command: "{{ container_command }} logout {{ zj_image.dest_registry }}"

roles/mirror-container-images/tasks/main.yaml

@@ -0,0 +1,5 @@
+- name: Pull and push images
+  with_items: "{{ mirror_container_images_images }}"
+  include_tasks: inner.yaml
+  loop_control:
+    loop_var: zj_image

test-playbooks/registry/test-mirror.yaml

@@ -0,0 +1,59 @@
+# Run the intermediate registry on this host.
+
+- hosts: intermediate-registry
+  name: Set up the intermediate registry and add a build
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Run the intermediate registry
+      include_role:
+        name: run-test-intermediate-registry
+    - name: Install the intermediate registry cert
+      include_role:
+        name: ensure-registry-cert
+      vars:
+        registry_host: localhost
+        registry_port: 5000
+        registry_cert: "{{ intermediate_registry_tls_cert }}"
+    - name: Set up user credentials for the intermediate registry
+      include_role:
+        name: intermediate-registry-user-config
+
+- hosts: builder
+  name: Test the container mirror role
+  tasks:
+    - name: Include intermediate registry vars
+      include_vars: vars/intermediate-registry-auth.yaml
+    - name: Set registry credentials
+      set_fact:
+        container_registry_credentials:
+          "zuul-jobs.intermediate-registry:5000":
+            username: "{{ intermediate_registry.username }}"
+            password: "{{ intermediate_registry.password }}"
+    - name: Configure /etc/hosts for intermediate registry
+      become: true
+      lineinfile:
+        path: /etc/hosts
+        state: present
+        regex: "^{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}$"
+        line: "{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}"
+        insertafter: EOF
+    - name: Install the intermediate registry cert
+      include_role:
+        name: ensure-registry-cert
+      vars:
+        registry_host: "{{ intermediate_registry.host }}"
+        registry_port: "{{ intermediate_registry.port }}"
+        registry_cert: "{{ intermediate_registry_tls_cert }}"
+    # This begins the simulation of what we would expect to happen in a
+    # normal job.
+    - name: Test the mirror role
+      include_role:
+        name: mirror-container-images
+      vars:
+        mirror_container_images_images:
+          - src_repository: quay.io/zuul-ci/zuul-client
+            src_tag: latest
+            dest_repository: "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/org/repo"
+            dest_tag: tag
+            dest_registry: "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}"

zuul-tests.d/container-roles-jobs.yaml

@@ -581,6 +581,27 @@
         - name: debian-bullseye
           label: debian-bullseye
 
+- job:
+    name: zuul-jobs-test-mirror-container-images
+    description: |
+      Test the mirror-container-images role.
+    files:
+      - roles/ensure-podman/.*
+      - test-playbooks/registry/.*
+      - roles/mirror-container-images/.*
+    pre-run: test-playbooks/registry/test-registry-pre.yaml
+    run: test-playbooks/registry/test-mirror.yaml
+    vars:
+      container_command: podman
+      multiarch: false
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-jammy
+        - name: builder
+          label: ubuntu-jammy
+
+
 # -* AUTOGENERATED *-
 #  The following project section is autogenerated by
 #    tox -e update-test-platforms
@@ -620,6 +641,7 @@
         - zuul-jobs-test-ensure-skopeo-ubuntu-jammy
         - zuul-jobs-test-ensure-skopeo-ubuntu-noble
         - zuul-jobs-test-ensure-podman-debian-bullseye
+        - zuul-jobs-test-mirror-container-images
     gate:
       jobs: *id001
     periodic-weekly:

zuul.d/container-jobs.yaml

@@ -29,3 +29,13 @@
     run: playbooks/container-image/promote.yaml
     nodeset:
       nodes: []
+
+- job:
+    name: mirror-container-images
+    description: |
+      Copy container images from one registry to another.
+
+      .. include:: ../../playbooks/container-image/README.rst
+      .. include:: ../../playbooks/container-image/credentials.rst
+    pre-run: playbooks/container-image/pre.yaml
+    run: playbooks/container-image/mirror.yaml