diff --git a/.ansible-lint b/.ansible-lint index 3a49204da..1a552bb7b 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -12,6 +12,7 @@ skip_list: - key-order # This suggests an arbitrary ordering of keys which means our developers would need to memorize someone's personal preference - name[play] # We have no objection to naming plays, but because of all of our short job playbooks, there are a *lot* of them that are pretty self-evident in the Zuul context, so it doesn't seem necessary. - name[casing] # We should use proper capitalization, but this is too nit-picky to waste time on + - name[template] # This is too simple of a check and does not allow for templates like "foo {{ bar }}-{{ baz }}", and is also fairly "personal preference" # NOTE(ianw) After following the jinja formatting extensions in # ansible-lint for a few releases, this does not seem to be diff --git a/doc/source/container-jobs.rst b/doc/source/container-jobs.rst index b19e9775a..e0c9ea711 100644 --- a/doc/source/container-jobs.rst +++ b/doc/source/container-jobs.rst @@ -4,3 +4,4 @@ Container Jobs .. zuul:autojob:: build-container-image .. zuul:autojob:: upload-container-image .. zuul:autojob:: promote-container-image +.. zuul:autojob:: mirror-container-images diff --git a/doc/source/container-roles.rst b/doc/source/container-roles.rst index dd4ce5784..23b50af02 100644 --- a/doc/source/container-roles.rst +++ b/doc/source/container-roles.rst @@ -11,6 +11,7 @@ Container Roles .. zuul:autorole:: ensure-podman .. zuul:autorole:: ensure-skopeo .. zuul:autorole:: ensure-quay-repo +.. zuul:autorole:: mirror-container-images .. zuul:autorole:: pause-buildset-registry .. zuul:autorole:: promote-container-image .. zuul:autorole:: promote-docker-image diff --git a/playbooks/container-image/README.rst b/playbooks/container-image/README.rst index 8ec4bc8af..84c385fbc 100644 --- a/playbooks/container-image/README.rst +++ b/playbooks/container-image/README.rst @@ -6,6 +6,10 @@ context: * :zuul:job:`upload-container-image`: Build and stage the images in a registry. * :zuul:job:`promote-container-image`: Promote previously uploaded images. +The following utility job may also be useful: + + * :zuul:job:`mirror-container-images`: Copy existing images from one registry to another. + The jobs can work in multiple modes depending on your requirements. They all accept the same input data, principally a list of dictionaries representing the images to build. YAML anchors_ can be diff --git a/playbooks/container-image/mirror.yaml b/playbooks/container-image/mirror.yaml new file mode 100644 index 000000000..35c1c1b96 --- /dev/null +++ b/playbooks/container-image/mirror.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - mirror-container-images diff --git a/roles/mirror-container-images/README.rst b/roles/mirror-container-images/README.rst new file mode 100644 index 000000000..97d1bdf91 --- /dev/null +++ b/roles/mirror-container-images/README.rst @@ -0,0 +1,72 @@ +Copy container images from one registry to another. + +.. zuul:rolevar:: mirror_container_images_images + :type: list + + A list of container images to copy. + + .. zuul:rolevar:: src_repository + :type: str + + The source image repository (including registry name). + + .. zuul:rolevar:: src_tag + :type: str + + The source image tag. + + .. zuul:rolevar:: dest_repository + :type: str + + The destination image repository (including registry name). + + .. zuul:rolevar:: dest_tag + :type: str + + The destination image tag. + + .. zuul:rolevar:: dest_registry + :type: str + + The name of the registry to which the destination image will be + pushed. + +.. zuul:rolevar:: container_registry_credentials + :type: dict + + This is expected to be a Zuul Secret in dictionary form. Each key + is the name of a registry, and its value a dictionary with + information about that registry. + + Example: + + .. code-block:: yaml + + container_registry_credentials: + quay.io: + username: foo + password: bar + + .. zuul:rolevar:: [registry name] + :type: dict + + Information about a registry. The key is the registry name, and + its value a dict as follows: + + .. zuul:rolevar:: username + + The registry username. + + .. zuul:rolevar:: password + + The registry password. + + .. zuul:rolevar:: repository + + Optional; if supplied this is a regular expression which + restricts to what repositories the image may be uploaded. The + following example allows projects to upload images to + repositories within an organization based on their own names:: + + repository: "^myorgname/{{ zuul.project.short_name }}.*" + diff --git a/roles/mirror-container-images/tasks/inner.yaml b/roles/mirror-container-images/tasks/inner.yaml new file mode 100644 index 000000000..779ee9dd3 --- /dev/null +++ b/roles/mirror-container-images/tasks/inner.yaml @@ -0,0 +1,31 @@ +- name: Verify repository names + when: | + container_registry_credentials is defined + and zj_image.dest_registry not in container_registry_credentials + fail: + msg: "{{ zj_image.dest_registry }} credentials not found" + +- name: Verify repository permission + when: | + container_registry_credentials[zj_image.dest_registry].repository is defined and + not zj_image.dest_repository | regex_search(container_registry_credentials[zj_image.dest_registry].repository) + fail: + msg: "{{ zj_image.repository }} not permitted by {{ container_registry_credentials[zj_image.dest_registry].repository }}" + +- name: Log in to registry + command: "{{ container_command }} login -u {{ container_registry_credentials[zj_image.dest_registry].username }} -p {{ container_registry_credentials[zj_image.dest_registry].password }} {{ zj_image.dest_registry }}" + no_log: true + +- name: Push and pull image + block: + - name: "Pull image {{ zj_image.src_repository }}:{{ zj_image.src_tag }}" + command: "{{ container_command }} pull {{ zj_image.src_repository }}:{{ zj_image.src_tag }}" + + - name: Retag image + command: "{{ container_command }} tag {{ zj_image.src_repository }}:{{ zj_image.src_tag }} {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}" + + - name: "Push image {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}" + command: "{{ container_command }} push {{ zj_image.dest_repository }}:{{ zj_image.dest_tag }}" + always: + - name: Log out of registry + command: "{{ container_command }} logout {{ zj_image.dest_registry }}" diff --git a/roles/mirror-container-images/tasks/main.yaml b/roles/mirror-container-images/tasks/main.yaml new file mode 100644 index 000000000..09435cb81 --- /dev/null +++ b/roles/mirror-container-images/tasks/main.yaml @@ -0,0 +1,5 @@ +- name: Pull and push images + with_items: "{{ mirror_container_images_images }}" + include_tasks: inner.yaml + loop_control: + loop_var: zj_image diff --git a/test-playbooks/registry/test-mirror.yaml b/test-playbooks/registry/test-mirror.yaml new file mode 100644 index 000000000..7496c4cd2 --- /dev/null +++ b/test-playbooks/registry/test-mirror.yaml @@ -0,0 +1,59 @@ +# Run the intermediate registry on this host. + +- hosts: intermediate-registry + name: Set up the intermediate registry and add a build + tasks: + - name: Include intermediate registry vars + include_vars: vars/intermediate-registry-auth.yaml + - name: Run the intermediate registry + include_role: + name: run-test-intermediate-registry + - name: Install the intermediate registry cert + include_role: + name: ensure-registry-cert + vars: + registry_host: localhost + registry_port: 5000 + registry_cert: "{{ intermediate_registry_tls_cert }}" + - name: Set up user credentials for the intermediate registry + include_role: + name: intermediate-registry-user-config + +- hosts: builder + name: Test the container mirror role + tasks: + - name: Include intermediate registry vars + include_vars: vars/intermediate-registry-auth.yaml + - name: Set registry credentials + set_fact: + container_registry_credentials: + "zuul-jobs.intermediate-registry:5000": + username: "{{ intermediate_registry.username }}" + password: "{{ intermediate_registry.password }}" + - name: Configure /etc/hosts for intermediate registry + become: true + lineinfile: + path: /etc/hosts + state: present + regex: "^{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}$" + line: "{{ hostvars['intermediate-registry'].nodepool.private_ipv4 }}\t{{ intermediate_registry.host }}" + insertafter: EOF + - name: Install the intermediate registry cert + include_role: + name: ensure-registry-cert + vars: + registry_host: "{{ intermediate_registry.host }}" + registry_port: "{{ intermediate_registry.port }}" + registry_cert: "{{ intermediate_registry_tls_cert }}" + # This begins the simulation of what we would expect to happen in a + # normal job. + - name: Test the mirror role + include_role: + name: mirror-container-images + vars: + mirror_container_images_images: + - src_repository: quay.io/zuul-ci/zuul-client + src_tag: latest + dest_repository: "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}/org/repo" + dest_tag: tag + dest_registry: "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}" diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml index 3925f8590..d4fbcbe09 100644 --- a/zuul-tests.d/container-roles-jobs.yaml +++ b/zuul-tests.d/container-roles-jobs.yaml @@ -581,6 +581,27 @@ - name: debian-bullseye label: debian-bullseye +- job: + name: zuul-jobs-test-mirror-container-images + description: | + Test the mirror-container-images role. + files: + - roles/ensure-podman/.* + - test-playbooks/registry/.* + - roles/mirror-container-images/.* + pre-run: test-playbooks/registry/test-registry-pre.yaml + run: test-playbooks/registry/test-mirror.yaml + vars: + container_command: podman + multiarch: false + nodeset: + nodes: + - name: intermediate-registry + label: ubuntu-jammy + - name: builder + label: ubuntu-jammy + + # -* AUTOGENERATED *- # The following project section is autogenerated by # tox -e update-test-platforms @@ -620,6 +641,7 @@ - zuul-jobs-test-ensure-skopeo-ubuntu-jammy - zuul-jobs-test-ensure-skopeo-ubuntu-noble - zuul-jobs-test-ensure-podman-debian-bullseye + - zuul-jobs-test-mirror-container-images gate: jobs: *id001 periodic-weekly: diff --git a/zuul.d/container-jobs.yaml b/zuul.d/container-jobs.yaml index 7d430d33d..df53eea29 100644 --- a/zuul.d/container-jobs.yaml +++ b/zuul.d/container-jobs.yaml @@ -29,3 +29,13 @@ run: playbooks/container-image/promote.yaml nodeset: nodes: [] + +- job: + name: mirror-container-images + description: | + Copy container images from one registry to another. + + .. include:: ../../playbooks/container-image/README.rst + .. include:: ../../playbooks/container-image/credentials.rst + pre-run: playbooks/container-image/pre.yaml + run: playbooks/container-image/mirror.yaml