Merge "Support multi-arch image builds with docker buildx"

roles/build-docker-image/common.rst

@@ -136,4 +136,15 @@ using this role.
 
       A list of labels to attach to the built image, in the form of "key=value".
 
+   .. zuul:rolevar:: arch
+      :type: list
+      :default: []
+
+      A list of architectures to build on. When enabling this on any
+      image, all of them will be built with ``docker buildx``.
+
+      Valid values are ``linux/amd64``, ``linux/arm64``, ``linux/riscv64``,
+      ``linux/ppc64le``, ``linux/s390x``, ``linux/386``,
+      ``linux/arm/v7``, ``linux/arm/v6``.
+
 .. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor//

roles/build-docker-image/tasks/build.yaml

@@ -1,49 +1,20 @@
-- name: Check sibling directory
-  stat:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-  register: _dot_zuul_siblings
-
-# This should have been cleaned up; multiple builds may specify
-# different siblings to include so we need to start fresh.
-- name: Check for clean build
-  assert:
-    that: not _dot_zuul_siblings.stat.exists
-
-- name: Create sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-    state: directory
-    mode: 0755
-  when: item.siblings is defined
-
-# NOTE(ianw): could use recursive copy: with remote_src, but it's
-# Ansible 2.8 only.  take the simple approach.
-- name: Copy sibling source directories
-  command:
-    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-    chdir: '~/src'
-  loop: '{{ item.siblings }}'
-  loop_control:
-    loop_var: zj_sibling
-  when: item.siblings is defined
-
 - name: Build a docker image
   command: >-
-    docker build {{ item.path | default('.') }} -f {{ item.dockerfile | default(docker_dockerfile) }}
+    docker build {{ zj_image.path | default('.') }} -f {{ zj_image.dockerfile | default(docker_dockerfile) }}
-    {% if item.target | default(false) -%}
+    {% if zj_image.target | default(false) -%}
-      --target {{ item.target }}
+      --target {{ zj_image.target }}
     {% endif -%}
-    {% for build_arg in item.build_args | default([]) -%}
+    {% for build_arg in zj_image.build_args | default([]) -%}
       --build-arg {{ build_arg }}
     {% endfor -%}
-    {% if item.siblings | default(false) -%}
+    {% if zj_image.siblings | default(false) -%}
-      --build-arg "ZUUL_SIBLINGS={{ item.siblings | join(' ') }}"
+      --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
     {% endif -%}
-    {% for tag in item.tags | default(['latest']) -%}
+    {% for tag in zj_image.tags | default(['latest']) -%}
       {% if zuul.change | default(false) -%}
-        --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }}
+        --tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
       {% endif -%}
-      --tag {{ item.repository }}:{{ tag }}
+      --tag {{ zj_image.repository }}:{{ tag }}
     {% endfor -%}
     {% for label in zj_image.labels | default([]) -%}
       --label "{{ label }}"
@@ -51,10 +22,4 @@
     --label "org.zuul-ci.change={{ zuul.change }}"
     --label "org.zuul-ci.change_url={{ zuul.change_url }}"
   args:
-    chdir: "{{ zuul_work_dir }}/{{ item.context }}"
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
-
-- name: Cleanup sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/.zuul-siblings'
-    state: absent
-

roles/build-docker-image/tasks/buildx.yaml

@@ -0,0 +1,43 @@
+- name: Build a docker image
+  command: >-
+    docker buildx build {{ zj_image.path | default('.') }} -f {{ zj_image.dockerfile | default(docker_dockerfile) }}
+      --platform={{ zj_image.arch | join(',') }}
+    {% if zj_image.target | default(false) -%}
+      --target {{ zj_image.target }}
+    {% endif -%}
+    {% for build_arg in zj_image.build_args | default([]) -%}
+      --build-arg {{ build_arg }}
+    {% endfor -%}
+    {% if zj_image.siblings | default(false) -%}
+      --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
+    {% endif -%}
+    {% for tag in zj_image.tags | default(['latest']) -%}
+      {% if zuul.change | default(false) -%}
+        --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
+      {% endif -%}
+      --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }}
+    {% endfor -%}
+    {% for label in zj_image.labels | default([]) -%}
+      --label "{{ label }}"
+    {% endfor %}
+    --label "org.zuul-ci.change={{ zuul.change }}"
+    --label "org.zuul-ci.change_url={{ zuul.change_url }}"
+    --push
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Pull images from buildset registry
+  command: >-
+    docker pull {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+
+- name: Tag image for local registry
+  command: >-
+    docker tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} {{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag

roles/build-docker-image/tasks/main.yaml

@@ -5,9 +5,11 @@
     buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
   ignore_errors: true
 
-- name: Build docker images
+- name: Set up siblings
-  include_tasks: build.yaml
+  include_tasks: siblings.yaml
   loop: "{{ docker_images }}"
+  loop_control:
+    loop_var: zj_image
 
 # Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
 # so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
@@ -28,10 +30,49 @@
   set_fact:
     buildset_registry_alias: "{{ buildset_registry.host }}"
   when: buildset_registry is defined and not ( buildset_registry.host | ipaddr )
-# Push each image.
+
-- name: Push image to buildset registry
+- name: Determine if we need to use buildx or normal build
-  when: buildset_registry is defined
+  set_fact:
-  include_tasks: push.yaml
+    use_buildx: "{{ docker_images | selectattr('arch', 'defined') | list }}"
-  loop: "{{ docker_images }}"
+
-  loop_control:
+- name: Normal docker block
-    loop_var: image
+  when: not use_buildx
+  block:
+
+    - name: Build docker images
+      include_tasks: build.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+    # Push each image.
+    - name: Push image to buildset registry
+      when: buildset_registry is defined
+      include_tasks: push.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Buildx block
+  when: use_buildx
+  block:
+
+    - name: Assert buildset registry is defined for buildx
+      assert:
+        that:
+           - buildset_registry is defined
+        fail_msg: "Building multi-arch images requires a buildset registry"
+
+    - name: Set up buildx builders
+      include_tasks: setup-buildx.yaml
+
+    - name: Build and push each image using buildx.
+      include_tasks: buildx.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Cleanup sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/.zuul-siblings'
+    state: absent

roles/build-docker-image/tasks/push.yaml

@@ -1,12 +1,12 @@
 - name: Tag image for buildset registry
   command: >-
-    docker tag {{ image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
+    docker tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
 - name: Push tag to buildset registry
   command: >-
-    docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
+    docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag

roles/build-docker-image/tasks/setup-buildx.yaml

@@ -0,0 +1,56 @@
+- name: Write buildkit.toml file
+  template:
+    dest: /tmp/buildkitd.toml
+    src: buildkitd.toml.j2
+
+- name: Run binfmt container
+  command: docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Create builder
+  command: docker buildx create --name mybuilder --driver-opt network=host --config /tmp/buildkitd.toml
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Use builder
+  command: docker buildx use mybuilder
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Bootstrap builder
+  command: docker buildx inspect --bootstrap
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Copy buildset registry TLS cert into worker container
+  command: "docker cp {{ ca_dir }}/buildset-registry.crt buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+
+- name: Update CA certs in worker container
+  command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+
+- name: Copy /etc/hosts for editing
+  command: docker cp buildx_buildkit_mybuilder0:/etc/hosts /tmp/mybuilder-hosts
+
+# Docker buildx has its own /etc/hosts in the builder image.
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  become: yes
+  lineinfile:
+    path: /tmp/mybuilder-hosts
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
+    insertafter: EOF
+  when: buildset_registry is defined and buildset_registry.host | ipaddr
+
+- name: Unmount the /etc/hosts mount
+  command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts
+
+# NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts
+# in the previous step, when we try to copy the file back directly, we get:
+#   unlinkat /etc/hosts: device or resource busy
+- name: Copy modified hosts file back in
+  command: docker cp /tmp/mybuilder-hosts buildx_buildkit_mybuilder0:/etc/new-hosts
+
+- name: Copy modified hosts file into place
+  command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts

roles/build-docker-image/tasks/siblings.yaml

@@ -0,0 +1,28 @@
+- name: Check sibling directory
+  stat:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+  register: _dot_zuul_siblings
+
+# This should have been cleaned up; multiple builds may specify
+# different siblings to include so we need to start fresh.
+- name: Check for clean build
+  assert:
+    that: not _dot_zuul_siblings.stat.exists
+
+- name: Create sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    state: directory
+    mode: 0755
+  when: zj_image.siblings is defined
+
+# NOTE(ianw): could use recursive copy: with remote_src, but it's
+# Ansible 2.8 only.  take the simple approach.
+- name: Copy sibling source directories
+  command:
+    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    chdir: '~/src'
+  loop: '{{ zj_image.siblings }}'
+  loop_control:
+    loop_var: zj_sibling
+  when: zj_image.siblings is defined

roles/build-docker-image/templates/buildkitd.toml.j2

@@ -0,0 +1,8 @@
+[registry."docker.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}"]
+
+[registry."quay.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/quay.io"]
+
+[registry."gcr.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/gcr.io"]

roles/build-docker-image/vars/CentOS.yaml

@@ -0,0 +1,2 @@
+ca_dir: /etc/pki/ca-trust/source/anchors
+ca_command: update-ca-trust

roles/build-docker-image/vars/default.yaml

@@ -0,0 +1,2 @@
+ca_dir: /usr/local/share/ca-certificates
+ca_command: update-ca-certificates

roles/run-buildset-registry/tasks/main.yaml

@@ -50,7 +50,7 @@
     --publish="1{{ buildset_registry_port }}:5000"
     --volume="{{ buildset_registry_root }}/tls:/tls"
     --volume="{{ buildset_registry_root }}/conf:/conf"
-    docker.io/zuul/zuul-registry:latest
+    docker.io/zuul/zuul-registry:latest zuul-registry -d
 
 # Start a socat tunnel to the buildset registry to work around
 # https://github.com/containers/libpod/issues/4311

test-playbooks/registry/docker-siblings/Dockerfile

@@ -1,6 +1,7 @@
-FROM docker.io/library/debian:testing
+FROM docker.io/upstream/image
 ARG ZUUL_SIBLINGS=""
 RUN echo "Zuul siblings: ${ZUUL_SIBLINGS}"
+RUN cp /test-nonce /test-nonce-is-there
 COPY .zuul-siblings/opendev.org/project/fake-sibling/file /target
 COPY .zuul-siblings/openstack.org/project/fake-sibling/file /target
 CMD echo "Zuul container test"; sleep infinity

test-playbooks/registry/docker/Dockerfile

@@ -1,2 +1,3 @@
 FROM docker.io/library/debian:testing
+RUN touch "/test-nonce"
 CMD echo "Zuul container test"; sleep infinity

test-playbooks/registry/test-registry.yaml

@@ -134,12 +134,20 @@
       include_role:
         name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
-        docker_images:
+        _normal_docker_images:
           - context: test-playbooks/registry/docker-siblings
             repository: downstream/image
             siblings:
               - opendev.org/project/fake-sibling
               - openstack.org/project/fake-sibling
+        _arch_docker_images:
+          - context: test-playbooks/registry/docker-siblings
+            repository: downstream/image
+            siblings:
+              - opendev.org/project/fake-sibling
+              - openstack.org/project/fake-sibling
+            arch: ['linux/amd64', 'linux/arm64']
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"
         container_images: "{{ docker_images }}"
 - hosts: executor
   name: Test pushing to the intermediate registry
@@ -150,9 +158,14 @@
       include_role:
         name: push-to-intermediate-registry
       vars:
-        docker_images:
+        _normal_docker_images:
           - context: playbooks/registry/docker
             repository: downstream/image
+        _arch_docker_images:
+          - context: playbooks/registry/docker
+            repository: downstream/image
+            arch: ['linux/amd64', 'linux/arm64']
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"
         container_images: "{{ docker_images }}"
 
 # And finally an external verification step.

zuul-tests.d/container-roles-jobs.yaml

@@ -84,6 +84,38 @@
     post-run: test-playbooks/registry/test-registry-post.yaml
     vars:
       container_command: docker
+      multiarch: false
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-bionic
+        - name: executor
+          label: ubuntu-bionic
+        - name: builder
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-registry-docker-multiarch
+    description: |
+      Test the intermediate registry roles with multiarch.
+
+      This job tests changes to the intermediate registry roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/ensure-docker/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    pre-run: test-playbooks/registry/test-registry-pre.yaml
+    run: test-playbooks/registry/test-registry.yaml
+    post-run: test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: docker
+      multiarch: true
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -115,6 +147,7 @@
     post-run: test-playbooks/registry/test-registry-post.yaml
     vars:
       container_command: podman
+      multiarch: false
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -321,6 +354,7 @@
         - zuul-jobs-test-ensure-docker-ubuntu-bionic
         - zuul-jobs-test-ensure-docker-ubuntu-xenial
         - zuul-jobs-test-registry-docker
+        - zuul-jobs-test-registry-docker-multiarch
         - zuul-jobs-test-registry-podman
         - zuul-jobs-test-registry-buildset-registry
         - zuul-jobs-test-registry-buildset-registry-k8s-docker