From 63bd307e6301089917b06eb3c3aa58ee5dcaa732 Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Thu, 23 Apr 2020 09:46:29 -0500
Subject: [PATCH] Support multi-arch image builds with docker buildx

Docker has experimental support for building multi-arch
container images with a buildx command. Currently it only
supports pushing to a registry after running and the images
don't end up in the local docker images list. To work around
that, push to the buildset registry then pull back. This
is the inverse of the normal case where we build, then
retag, then push. The end result should be the same.

Change-Id: I6a4c4f9e262add909d2d5c2efa33ec69b9d9364a
---
 roles/build-docker-image/common.rst           | 11 ++++
 roles/build-docker-image/tasks/build.yaml     | 55 ++++-------------
 roles/build-docker-image/tasks/buildx.yaml    | 43 ++++++++++++++
 roles/build-docker-image/tasks/main.yaml      | 59 ++++++++++++++++---
 roles/build-docker-image/tasks/push.yaml      |  8 +--
 .../tasks/setup-buildx.yaml                   | 56 ++++++++++++++++++
 roles/build-docker-image/tasks/siblings.yaml  | 28 +++++++++
 .../templates/buildkitd.toml.j2               |  8 +++
 roles/build-docker-image/vars/CentOS.yaml     |  2 +
 roles/build-docker-image/vars/default.yaml    |  2 +
 roles/run-buildset-registry/tasks/main.yaml   |  2 +-
 .../registry/docker-siblings/Dockerfile       |  3 +-
 test-playbooks/registry/docker/Dockerfile     |  1 +
 test-playbooks/registry/test-registry.yaml    | 17 +++++-
 zuul-tests.d/container-roles-jobs.yaml        | 34 +++++++++++
 15 files changed, 267 insertions(+), 62 deletions(-)
 create mode 100644 roles/build-docker-image/tasks/buildx.yaml
 create mode 100644 roles/build-docker-image/tasks/setup-buildx.yaml
 create mode 100644 roles/build-docker-image/tasks/siblings.yaml
 create mode 100644 roles/build-docker-image/templates/buildkitd.toml.j2
 create mode 100644 roles/build-docker-image/vars/CentOS.yaml
 create mode 100644 roles/build-docker-image/vars/default.yaml

diff --git a/roles/build-docker-image/common.rst b/roles/build-docker-image/common.rst
index 7b04e0441..10753e62e 100644
--- a/roles/build-docker-image/common.rst
+++ b/roles/build-docker-image/common.rst
@@ -136,4 +136,15 @@ using this role.
 
       A list of labels to attach to the built image, in the form of "key=value".
 
+   .. zuul:rolevar:: arch
+      :type: list
+      :default: []
+
+      A list of architectures to build on. When enabling this on any
+      image, all of them will be built with ``docker buildx``.
+
+      Valid values are ``linux/amd64``, ``linux/arm64``, ``linux/riscv64``,
+      ``linux/ppc64le``, ``linux/s390x``, ``linux/386``,
+      ``linux/arm/v7``, ``linux/arm/v6``.
+
 .. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor//
diff --git a/roles/build-docker-image/tasks/build.yaml b/roles/build-docker-image/tasks/build.yaml
index 5ccea855b..7aa2e38cb 100644
--- a/roles/build-docker-image/tasks/build.yaml
+++ b/roles/build-docker-image/tasks/build.yaml
@@ -1,49 +1,20 @@
-- name: Check sibling directory
-  stat:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-  register: _dot_zuul_siblings
-
-# This should have been cleaned up; multiple builds may specify
-# different siblings to include so we need to start fresh.
-- name: Check for clean build
-  assert:
-    that: not _dot_zuul_siblings.stat.exists
-
-- name: Create sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-    state: directory
-    mode: 0755
-  when: item.siblings is defined
-
-# NOTE(ianw): could use recursive copy: with remote_src, but it's
-# Ansible 2.8 only.  take the simple approach.
-- name: Copy sibling source directories
-  command:
-    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
-    chdir: '~/src'
-  loop: '{{ item.siblings }}'
-  loop_control:
-    loop_var: zj_sibling
-  when: item.siblings is defined
-
 - name: Build a docker image
   command: >-
-    docker build {{ item.path | default('.') }} -f {{ item.dockerfile | default(docker_dockerfile) }}
-    {% if item.target | default(false) -%}
-      --target {{ item.target }}
+    docker build {{ zj_image.path | default('.') }} -f {{ zj_image.dockerfile | default(docker_dockerfile) }}
+    {% if zj_image.target | default(false) -%}
+      --target {{ zj_image.target }}
     {% endif -%}
-    {% for build_arg in item.build_args | default([]) -%}
+    {% for build_arg in zj_image.build_args | default([]) -%}
       --build-arg {{ build_arg }}
     {% endfor -%}
-    {% if item.siblings | default(false) -%}
-      --build-arg "ZUUL_SIBLINGS={{ item.siblings | join(' ') }}"
+    {% if zj_image.siblings | default(false) -%}
+      --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
     {% endif -%}
-    {% for tag in item.tags | default(['latest']) -%}
+    {% for tag in zj_image.tags | default(['latest']) -%}
       {% if zuul.change | default(false) -%}
-        --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }}
+        --tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
       {% endif -%}
-      --tag {{ item.repository }}:{{ tag }}
+      --tag {{ zj_image.repository }}:{{ tag }}
     {% endfor -%}
     {% for label in zj_image.labels | default([]) -%}
       --label "{{ label }}"
@@ -51,10 +22,4 @@
     --label "org.zuul-ci.change={{ zuul.change }}"
     --label "org.zuul-ci.change_url={{ zuul.change_url }}"
   args:
-    chdir: "{{ zuul_work_dir }}/{{ item.context }}"
-
-- name: Cleanup sibling source directory
-  file:
-    path: '{{ zuul_work_dir }}/.zuul-siblings'
-    state: absent
-
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
diff --git a/roles/build-docker-image/tasks/buildx.yaml b/roles/build-docker-image/tasks/buildx.yaml
new file mode 100644
index 000000000..2e21d6063
--- /dev/null
+++ b/roles/build-docker-image/tasks/buildx.yaml
@@ -0,0 +1,43 @@
+- name: Build a docker image
+  command: >-
+    docker buildx build {{ zj_image.path | default('.') }} -f {{ zj_image.dockerfile | default(docker_dockerfile) }}
+      --platform={{ zj_image.arch | join(',') }}
+    {% if zj_image.target | default(false) -%}
+      --target {{ zj_image.target }}
+    {% endif -%}
+    {% for build_arg in zj_image.build_args | default([]) -%}
+      --build-arg {{ build_arg }}
+    {% endfor -%}
+    {% if zj_image.siblings | default(false) -%}
+      --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
+    {% endif -%}
+    {% for tag in zj_image.tags | default(['latest']) -%}
+      {% if zuul.change | default(false) -%}
+        --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
+      {% endif -%}
+      --tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ tag }}
+    {% endfor -%}
+    {% for label in zj_image.labels | default([]) -%}
+      --label "{{ label }}"
+    {% endfor %}
+    --label "org.zuul-ci.change={{ zuul.change }}"
+    --label "org.zuul-ci.change_url={{ zuul.change_url }}"
+    --push
+  args:
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Pull images from buildset registry
+  command: >-
+    docker pull {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
+
+- name: Tag image for local registry
+  command: >-
+    docker tag {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }} {{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
+  loop_control:
+    loop_var: zj_image_tag
diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml
index c5d089880..5ab2fb9aa 100644
--- a/roles/build-docker-image/tasks/main.yaml
+++ b/roles/build-docker-image/tasks/main.yaml
@@ -5,9 +5,11 @@
     buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
   ignore_errors: true
 
-- name: Build docker images
-  include_tasks: build.yaml
+- name: Set up siblings
+  include_tasks: siblings.yaml
   loop: "{{ docker_images }}"
+  loop_control:
+    loop_var: zj_image
 
 # Docker doesn't understand docker push [1234:5678::]:5000/image/path:tag
 # so we set up /etc/hosts with a registry alias name to support ipv6 and 4.
@@ -28,10 +30,49 @@
   set_fact:
     buildset_registry_alias: "{{ buildset_registry.host }}"
   when: buildset_registry is defined and not ( buildset_registry.host | ipaddr )
-# Push each image.
-- name: Push image to buildset registry
-  when: buildset_registry is defined
-  include_tasks: push.yaml
-  loop: "{{ docker_images }}"
-  loop_control:
-    loop_var: image
+
+- name: Determine if we need to use buildx or normal build
+  set_fact:
+    use_buildx: "{{ docker_images | selectattr('arch', 'defined') | list }}"
+
+- name: Normal docker block
+  when: not use_buildx
+  block:
+
+    - name: Build docker images
+      include_tasks: build.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+    # Push each image.
+    - name: Push image to buildset registry
+      when: buildset_registry is defined
+      include_tasks: push.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Buildx block
+  when: use_buildx
+  block:
+
+    - name: Assert buildset registry is defined for buildx
+      assert:
+        that:
+           - buildset_registry is defined
+        fail_msg: "Building multi-arch images requires a buildset registry"
+
+    - name: Set up buildx builders
+      include_tasks: setup-buildx.yaml
+
+    - name: Build and push each image using buildx.
+      include_tasks: buildx.yaml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
+
+- name: Cleanup sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/.zuul-siblings'
+    state: absent
diff --git a/roles/build-docker-image/tasks/push.yaml b/roles/build-docker-image/tasks/push.yaml
index 8b17df674..8364c3951 100644
--- a/roles/build-docker-image/tasks/push.yaml
+++ b/roles/build-docker-image/tasks/push.yaml
@@ -1,12 +1,12 @@
 - name: Tag image for buildset registry
   command: >-
-    docker tag {{ image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+    docker tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
 - name: Push tag to buildset registry
   command: >-
-    docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+    docker push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
diff --git a/roles/build-docker-image/tasks/setup-buildx.yaml b/roles/build-docker-image/tasks/setup-buildx.yaml
new file mode 100644
index 000000000..ac1d15aaa
--- /dev/null
+++ b/roles/build-docker-image/tasks/setup-buildx.yaml
@@ -0,0 +1,56 @@
+- name: Write buildkit.toml file
+  template:
+    dest: /tmp/buildkitd.toml
+    src: buildkitd.toml.j2
+
+- name: Run binfmt container
+  command: docker run --rm --privileged docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Create builder
+  command: docker buildx create --name mybuilder --driver-opt network=host --config /tmp/buildkitd.toml
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Use builder
+  command: docker buildx use mybuilder
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Bootstrap builder
+  command: docker buildx inspect --bootstrap
+  environment:
+    DOCKER_CLI_EXPERIMENTAL: enabled
+
+- name: Copy buildset registry TLS cert into worker container
+  command: "docker cp {{ ca_dir }}/buildset-registry.crt buildx_buildkit_mybuilder0:/usr/local/share/ca-certificates"
+
+- name: Update CA certs in worker container
+  command: docker exec buildx_buildkit_mybuilder0 update-ca-certificates
+
+- name: Copy /etc/hosts for editing
+  command: docker cp buildx_buildkit_mybuilder0:/etc/hosts /tmp/mybuilder-hosts
+
+# Docker buildx has its own /etc/hosts in the builder image.
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  become: yes
+  lineinfile:
+    path: /tmp/mybuilder-hosts
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
+    insertafter: EOF
+  when: buildset_registry is defined and buildset_registry.host | ipaddr
+
+- name: Unmount the /etc/hosts mount
+  command: docker exec buildx_buildkit_mybuilder0 umount /etc/hosts
+
+# NOTE(mordred) This is done in two steps. Even though we've unmounted /etc/hosts
+# in the previous step, when we try to copy the file back directly, we get:
+#   unlinkat /etc/hosts: device or resource busy
+- name: Copy modified hosts file back in
+  command: docker cp /tmp/mybuilder-hosts buildx_buildkit_mybuilder0:/etc/new-hosts
+
+- name: Copy modified hosts file into place
+  command: docker exec buildx_buildkit_mybuilder0 cp /etc/new-hosts /etc/hosts
diff --git a/roles/build-docker-image/tasks/siblings.yaml b/roles/build-docker-image/tasks/siblings.yaml
new file mode 100644
index 000000000..896f96387
--- /dev/null
+++ b/roles/build-docker-image/tasks/siblings.yaml
@@ -0,0 +1,28 @@
+- name: Check sibling directory
+  stat:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+  register: _dot_zuul_siblings
+
+# This should have been cleaned up; multiple builds may specify
+# different siblings to include so we need to start fresh.
+- name: Check for clean build
+  assert:
+    that: not _dot_zuul_siblings.stat.exists
+
+- name: Create sibling source directory
+  file:
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    state: directory
+    mode: 0755
+  when: zj_image.siblings is defined
+
+# NOTE(ianw): could use recursive copy: with remote_src, but it's
+# Ansible 2.8 only.  take the simple approach.
+- name: Copy sibling source directories
+  command:
+    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
+    chdir: '~/src'
+  loop: '{{ zj_image.siblings }}'
+  loop_control:
+    loop_var: zj_sibling
+  when: zj_image.siblings is defined
diff --git a/roles/build-docker-image/templates/buildkitd.toml.j2 b/roles/build-docker-image/templates/buildkitd.toml.j2
new file mode 100644
index 000000000..61d6f37ac
--- /dev/null
+++ b/roles/build-docker-image/templates/buildkitd.toml.j2
@@ -0,0 +1,8 @@
+[registry."docker.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}"]
+
+[registry."quay.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/quay.io"]
+
+[registry."gcr.io"]
+  mirrors = ["{{ buildset_registry_alias }}:{{ buildset_registry.port }}/gcr.io"]
diff --git a/roles/build-docker-image/vars/CentOS.yaml b/roles/build-docker-image/vars/CentOS.yaml
new file mode 100644
index 000000000..c2b260ab2
--- /dev/null
+++ b/roles/build-docker-image/vars/CentOS.yaml
@@ -0,0 +1,2 @@
+ca_dir: /etc/pki/ca-trust/source/anchors
+ca_command: update-ca-trust
diff --git a/roles/build-docker-image/vars/default.yaml b/roles/build-docker-image/vars/default.yaml
new file mode 100644
index 000000000..7bea1b23b
--- /dev/null
+++ b/roles/build-docker-image/vars/default.yaml
@@ -0,0 +1,2 @@
+ca_dir: /usr/local/share/ca-certificates
+ca_command: update-ca-certificates
diff --git a/roles/run-buildset-registry/tasks/main.yaml b/roles/run-buildset-registry/tasks/main.yaml
index 11504c4a1..bae986e37 100644
--- a/roles/run-buildset-registry/tasks/main.yaml
+++ b/roles/run-buildset-registry/tasks/main.yaml
@@ -50,7 +50,7 @@
     --publish="1{{ buildset_registry_port }}:5000"
     --volume="{{ buildset_registry_root }}/tls:/tls"
     --volume="{{ buildset_registry_root }}/conf:/conf"
-    docker.io/zuul/zuul-registry:latest
+    docker.io/zuul/zuul-registry:latest zuul-registry -d
 
 # Start a socat tunnel to the buildset registry to work around
 # https://github.com/containers/libpod/issues/4311
diff --git a/test-playbooks/registry/docker-siblings/Dockerfile b/test-playbooks/registry/docker-siblings/Dockerfile
index 8907b1e30..18e777555 100644
--- a/test-playbooks/registry/docker-siblings/Dockerfile
+++ b/test-playbooks/registry/docker-siblings/Dockerfile
@@ -1,6 +1,7 @@
-FROM docker.io/library/debian:testing
+FROM docker.io/upstream/image
 ARG ZUUL_SIBLINGS=""
 RUN echo "Zuul siblings: ${ZUUL_SIBLINGS}"
+RUN cp /test-nonce /test-nonce-is-there
 COPY .zuul-siblings/opendev.org/project/fake-sibling/file /target
 COPY .zuul-siblings/openstack.org/project/fake-sibling/file /target
 CMD echo "Zuul container test"; sleep infinity
diff --git a/test-playbooks/registry/docker/Dockerfile b/test-playbooks/registry/docker/Dockerfile
index 178d518e8..d32f9e817 100644
--- a/test-playbooks/registry/docker/Dockerfile
+++ b/test-playbooks/registry/docker/Dockerfile
@@ -1,2 +1,3 @@
 FROM docker.io/library/debian:testing
+RUN touch "/test-nonce"
 CMD echo "Zuul container test"; sleep infinity
diff --git a/test-playbooks/registry/test-registry.yaml b/test-playbooks/registry/test-registry.yaml
index b45734ab8..727334e3a 100644
--- a/test-playbooks/registry/test-registry.yaml
+++ b/test-playbooks/registry/test-registry.yaml
@@ -134,12 +134,20 @@
       include_role:
         name: "build-{{ (container_command == 'docker') | ternary('docker', 'container') }}-image"
       vars:
-        docker_images:
+        _normal_docker_images:
           - context: test-playbooks/registry/docker-siblings
             repository: downstream/image
             siblings:
               - opendev.org/project/fake-sibling
               - openstack.org/project/fake-sibling
+        _arch_docker_images:
+          - context: test-playbooks/registry/docker-siblings
+            repository: downstream/image
+            siblings:
+              - opendev.org/project/fake-sibling
+              - openstack.org/project/fake-sibling
+            arch: ['linux/amd64', 'linux/arm64']
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"
         container_images: "{{ docker_images }}"
 - hosts: executor
   name: Test pushing to the intermediate registry
@@ -150,9 +158,14 @@
       include_role:
         name: push-to-intermediate-registry
       vars:
-        docker_images:
+        _normal_docker_images:
           - context: playbooks/registry/docker
             repository: downstream/image
+        _arch_docker_images:
+          - context: playbooks/registry/docker
+            repository: downstream/image
+            arch: ['linux/amd64', 'linux/arm64']
+        docker_images: "{{ multiarch | ternary(_arch_docker_images, _normal_docker_images) }}"
         container_images: "{{ docker_images }}"
 
 # And finally an external verification step.
diff --git a/zuul-tests.d/container-roles-jobs.yaml b/zuul-tests.d/container-roles-jobs.yaml
index be062dc4c..c6c81e55c 100644
--- a/zuul-tests.d/container-roles-jobs.yaml
+++ b/zuul-tests.d/container-roles-jobs.yaml
@@ -82,6 +82,38 @@
     post-run: test-playbooks/registry/test-registry-post.yaml
     vars:
       container_command: docker
+      multiarch: false
+    nodeset:
+      nodes:
+        - name: intermediate-registry
+          label: ubuntu-bionic
+        - name: executor
+          label: ubuntu-bionic
+        - name: builder
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-registry-docker-multiarch
+    description: |
+      Test the intermediate registry roles with multiarch.
+
+      This job tests changes to the intermediate registry roles.  It
+      is not meant to be used directly but rather run on changes to
+      roles in the zuul-jobs repo.
+    files:
+      - roles/pull-from-intermediate-registry/.*
+      - roles/push-to-intermediate-registry/.*
+      - roles/ensure-docker/.*
+      - roles/build-docker-image/.*
+      - roles/run-buildset-registry/.*
+      - roles/use-buildset-registry/.*
+      - test-playbooks/registry/.*
+    pre-run: test-playbooks/registry/test-registry-pre.yaml
+    run: test-playbooks/registry/test-registry.yaml
+    post-run: test-playbooks/registry/test-registry-post.yaml
+    vars:
+      container_command: docker
+      multiarch: true
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -112,6 +144,7 @@
     post-run: test-playbooks/registry/test-registry-post.yaml
     vars:
       container_command: podman
+      multiarch: false
     nodeset:
       nodes:
         - name: intermediate-registry
@@ -311,6 +344,7 @@
         - zuul-jobs-test-ensure-docker-ubuntu-bionic
         - zuul-jobs-test-ensure-docker-ubuntu-xenial
         - zuul-jobs-test-registry-docker
+        - zuul-jobs-test-registry-docker-multiarch
         - zuul-jobs-test-registry-podman
         - zuul-jobs-test-registry-buildset-registry
         - zuul-jobs-test-registry-buildset-registry-k8s-docker