Configure containerd mirrors for multinode tests

The compute-kit jobs are used to test new images
which are published to buildset registry. We have
to configure containerd which is used for multinode
compute-kit jobs to use this buildset registry.

The role use-buildset-registry that we used before
does not properly configure containerd. So we
extended deploy-docker playbook to configure
both buildset registry and registry mirror
if they are defined.

Change-Id: Idb892a3fcaf51385998d466dbdff8de36d9dd338

tools/gate/playbooks/buildset_registry_alias.yaml

@@ -0,0 +1,19 @@
+- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses
+  lineinfile:
+    path: /etc/hosts
+    state: present
+    regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$"
+    line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
+    insertafter: EOF
+  when:
+    - buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using ip
+  set_fact:
+    buildset_registry_alias: zuul-jobs.buildset-registry
+  when:
+    - buildset_registry.host | ipaddr
+- name: Set buildset_registry alias variable when using name
+  set_fact:
+    buildset_registry_alias: "{{ buildset_registry.host }}"
+  when:
+    - not ( buildset_registry.host | ipaddr )

tools/gate/playbooks/deploy-docker.yaml

@@ -48,25 +48,103 @@
         state: present
         update_cache: true
 
+    - name: Install Crictl
+      shell: |
+        wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz
+        sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin
+        rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz
+      args:
+        executable: /bin/bash
+
     - name: Configure Docker daemon
       copy:
         src: files/daemon.json
         dest: /etc/docker/daemon.json
 
-    - name: Remove /etc/containerd/config.toml
-      file:
-        path: /etc/containerd/config.toml
-        state: absent
-      ignore_errors: true
-
-    - name: Restart containerd
-      service:
-        name: containerd
-        daemon_reload: yes
-        state: restarted
-
     - name: Restart docker
       service:
         name: docker
         daemon_reload: yes
         state: restarted
+
+    - name: Set mirror_fqdn fact
+      when:
+        - registry_mirror is not defined
+        - zuul_site_mirror_fqdn is defined
+      set_fact:
+        registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082"
+
+    - name: Set regitstry namespaces
+      set_fact:
+        registry_namespaces:
+          - namespace: "_default"
+            mirror: "{{ registry_mirror }}"
+            skip_server: true
+            skip_verify: true
+      when: registry_mirror is defined
+
+    - name: Buildset registry namespace
+      when: buildset_registry is defined
+      block:
+        - name: Buildset registry alias
+          include_tasks:
+            file: buildset_registry_alias.yaml
+
+        - name: Write buildset registry TLS certificate
+          copy:
+            content: "{{ buildset_registry.cert }}"
+            dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+            mode: 0644
+          register: buildset_registry_tls_ca
+
+        - name: Update CA certs
+          command: "update-ca-certificates"
+          when: buildset_registry_tls_ca is changed
+
+        - name: Set buildset registry namespace
+          set_fact:
+            buildset_registry_namespace:
+              namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+              mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+              ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+              auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}"
+
+        - name: Init registry_namespaces if not defined
+          set_fact:
+            registry_namespaces: "[]"
+          when: not registry_namespaces is defined
+
+        - name: Append buildset_registry to registry namespaces
+          when:
+            - buildset_registry_namespace is defined
+            - registry_namespaces is defined
+          set_fact:
+            registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}"
+
+    - name: Configure containerd
+      template:
+        src: files/containerd_config.toml
+        dest: /etc/containerd/config.toml
+
+    - name: Create containerd config directory hierarchy
+      file:
+        state: directory
+        path: /etc/containerd/certs.d
+
+    - name: Create host namespace directory
+      file:
+        state: directory
+        path: "/etc/containerd/certs.d/{{ item.namespace }}"
+      loop: "{{ registry_namespaces }}"
+
+    - name: Create hosts.toml file
+      template:
+        src: files/hosts.toml
+        dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml"
+      loop: "{{ registry_namespaces }}"
+
+    - name: Restart containerd
+      service:
+        name: containerd
+        daemon_reload: yes
+        state: restarted

tools/gate/playbooks/deploy-k8s.yaml

@@ -118,6 +118,7 @@
         executable: /bin/bash
 
 - hosts: all
+  become: true
   tasks:
     # We download Calico manifest on all nodes because we then want to download
     # Calico images BEFORE deploying it
@@ -132,7 +133,9 @@
     # for `k8s-app=kube-dns` isn't reached by slow download speeds
     - name: Download Calico images
       shell: |
-        awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {}
+        export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
+        export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
+        awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
       args:
         executable: /bin/bash
 

tools/gate/playbooks/files/containerd_config.toml

@@ -0,0 +1,11 @@
+version = 2
+disabled_plugins = []
+[plugins."io.containerd.grpc.v1.cri".registry]
+config_path = "/etc/containerd/certs.d"
+
+{% for item in registry_namespaces %}
+{% if item.auth is defined %}
+[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth]
+auth = "{{ item.auth }}"
+{% endif %}
+{% endfor %}

tools/gate/playbooks/files/hosts.toml

@@ -0,0 +1,12 @@
+{% if item.skip_server is not defined or not item.skip_server %}
+server = "{{ item.server | default('https://' + item.namespace) }}"
+{% endif %}
+
+[host."{{ item.mirror }}"]
+capabilities = ["pull", "resolve", "push"]
+{% if item.ca is defined %}
+ca = "{{ item.ca }}"
+{% endif %}
+{% if item.skip_verify %}
+skip_verify = true
+{% endif %}

tools/gate/playbooks/run-scripts.yaml

@@ -11,18 +11,48 @@
 # limitations under the License.
 
 ---
-- hosts: all
-  tasks:
-    - name: Override images
-      include_role:
-        name: override-images
-      when: buildset_registry is defined
-    - name: Use docker mirror
-      include_role:
-        name: use-docker-mirror
-
 - hosts: primary
   tasks:
+    - name: Override images
+      when: buildset_registry is defined
+      vars:
+        work_dir: "{{ zuul.project.src_dir }}"
+      block:
+        - name: Buildset registry alias
+          include_tasks:
+            file: buildset_registry_alias.yaml
+
+        - name: Print zuul
+          debug:
+            var: zuul
+
+        - name: Override proposed images from artifacts
+          shell: >
+            find {{ override_paths | join(" ") }} -type f -exec sed -Ei
+            "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
+          loop: "{{ zuul.artifacts | default([]) }}"
+          args:
+            chdir: "{{ work_dir }}"
+          loop_control:
+            loop_var: zj_zuul_artifact
+          when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
+          vars:
+            tag: "{{ zj_zuul_artifact.metadata.tag }}"
+            repo: "{{ zj_zuul_artifact.metadata.repository }}"
+            override_paths:
+              - ../openstack-helm*/*/values*
+              - ../openstack-helm-infra/tools/deployment/
+
+        - name: Diff
+          shell: |
+              set -ex;
+              for dir in openstack-helm openstack-helm-infra; do
+                path="{{ work_dir }}/../${dir}/"
+                if [ ! -d "${path}" ]; then continue; fi
+                echo "${dir} diff"
+                cd "${path}"; git diff; cd -;
+              done
+
     - name: "creating directory for run artifacts"
       file:
         path: "/tmp/artifacts"

zuul.d/jobs-openstack-helm.yaml

@@ -69,6 +69,7 @@
       calico_version: "v3.25"
       helm_version: "v3.6.3"
       yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
       zuul_osh_infra_relative_path: ../openstack-helm-infra
       gate_scripts_relative_path: ../openstack-helm