From c39638a1482339135f81f38058a4d1da29256eb1 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Fri, 11 Aug 2023 14:41:57 +0300 Subject: [PATCH] Configure containerd mirrors for multinode tests The compute-kit jobs are used to test new images which are published to buildset registry. We have to configure containerd which is used for multinode compute-kit jobs to use this buildset registry. The role use-buildset-registry that we used before does not properly configure containerd. So we extended deploy-docker playbook to configure both buildset registry and registry mirror if they are defined. Change-Id: Idb892a3fcaf51385998d466dbdff8de36d9dd338 --- .../playbooks/buildset_registry_alias.yaml | 19 ++++ tools/gate/playbooks/deploy-docker.yaml | 102 +++++++++++++++--- tools/gate/playbooks/deploy-k8s.yaml | 5 +- .../playbooks/files/containerd_config.toml | 11 ++ tools/gate/playbooks/files/hosts.toml | 12 +++ tools/gate/playbooks/run-scripts.yaml | 50 +++++++-- zuul.d/jobs-openstack-helm.yaml | 1 + 7 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 tools/gate/playbooks/buildset_registry_alias.yaml create mode 100644 tools/gate/playbooks/files/containerd_config.toml create mode 100644 tools/gate/playbooks/files/hosts.toml diff --git a/tools/gate/playbooks/buildset_registry_alias.yaml b/tools/gate/playbooks/buildset_registry_alias.yaml new file mode 100644 index 0000000000..0afadc1cd9 --- /dev/null +++ b/tools/gate/playbooks/buildset_registry_alias.yaml @@ -0,0 +1,19 @@ +- name: Configure /etc/hosts for buildset_registry to workaround docker not understanding ipv6 addresses + lineinfile: + path: /etc/hosts + state: present + regex: "^{{ buildset_registry.host }}\tzuul-jobs.buildset-registry$" + line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry" + insertafter: EOF + when: + - buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using ip + set_fact: + buildset_registry_alias: zuul-jobs.buildset-registry + when: + - buildset_registry.host | ipaddr +- name: Set buildset_registry alias variable when using name + set_fact: + buildset_registry_alias: "{{ buildset_registry.host }}" + when: + - not ( buildset_registry.host | ipaddr ) diff --git a/tools/gate/playbooks/deploy-docker.yaml b/tools/gate/playbooks/deploy-docker.yaml index ba4b704ad7..447f14923e 100644 --- a/tools/gate/playbooks/deploy-docker.yaml +++ b/tools/gate/playbooks/deploy-docker.yaml @@ -48,25 +48,103 @@ state: present update_cache: true + - name: Install Crictl + shell: | + wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz + sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin + rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz + args: + executable: /bin/bash + - name: Configure Docker daemon copy: src: files/daemon.json dest: /etc/docker/daemon.json - - name: Remove /etc/containerd/config.toml - file: - path: /etc/containerd/config.toml - state: absent - ignore_errors: true - - - name: Restart containerd - service: - name: containerd - daemon_reload: yes - state: restarted - - name: Restart docker service: name: docker daemon_reload: yes state: restarted + + - name: Set mirror_fqdn fact + when: + - registry_mirror is not defined + - zuul_site_mirror_fqdn is defined + set_fact: + registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082" + + - name: Set regitstry namespaces + set_fact: + registry_namespaces: + - namespace: "_default" + mirror: "{{ registry_mirror }}" + skip_server: true + skip_verify: true + when: registry_mirror is defined + + - name: Buildset registry namespace + when: buildset_registry is defined + block: + - name: Buildset registry alias + include_tasks: + file: buildset_registry_alias.yaml + + - name: Write buildset registry TLS certificate + copy: + content: "{{ buildset_registry.cert }}" + dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt" + mode: 0644 + register: buildset_registry_tls_ca + + - name: Update CA certs + command: "update-ca-certificates" + when: buildset_registry_tls_ca is changed + + - name: Set buildset registry namespace + set_fact: + buildset_registry_namespace: + namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}' + mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}' + ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt" + auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}" + + - name: Init registry_namespaces if not defined + set_fact: + registry_namespaces: "[]" + when: not registry_namespaces is defined + + - name: Append buildset_registry to registry namespaces + when: + - buildset_registry_namespace is defined + - registry_namespaces is defined + set_fact: + registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}" + + - name: Configure containerd + template: + src: files/containerd_config.toml + dest: /etc/containerd/config.toml + + - name: Create containerd config directory hierarchy + file: + state: directory + path: /etc/containerd/certs.d + + - name: Create host namespace directory + file: + state: directory + path: "/etc/containerd/certs.d/{{ item.namespace }}" + loop: "{{ registry_namespaces }}" + + - name: Create hosts.toml file + template: + src: files/hosts.toml + dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml" + loop: "{{ registry_namespaces }}" + + - name: Restart containerd + service: + name: containerd + daemon_reload: yes + state: restarted diff --git a/tools/gate/playbooks/deploy-k8s.yaml b/tools/gate/playbooks/deploy-k8s.yaml index 7994632e28..ab6c2492fa 100644 --- a/tools/gate/playbooks/deploy-k8s.yaml +++ b/tools/gate/playbooks/deploy-k8s.yaml @@ -118,6 +118,7 @@ executable: /bin/bash - hosts: all + become: true tasks: # We download Calico manifest on all nodes because we then want to download # Calico images BEFORE deploying it @@ -132,7 +133,9 @@ # for `k8s-app=kube-dns` isn't reached by slow download speeds - name: Download Calico images shell: | - awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {} + export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock + export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock + awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {} args: executable: /bin/bash diff --git a/tools/gate/playbooks/files/containerd_config.toml b/tools/gate/playbooks/files/containerd_config.toml new file mode 100644 index 0000000000..cc6ab0bc44 --- /dev/null +++ b/tools/gate/playbooks/files/containerd_config.toml @@ -0,0 +1,11 @@ +version = 2 +disabled_plugins = [] +[plugins."io.containerd.grpc.v1.cri".registry] +config_path = "/etc/containerd/certs.d" + +{% for item in registry_namespaces %} +{% if item.auth is defined %} +[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth] +auth = "{{ item.auth }}" +{% endif %} +{% endfor %} \ No newline at end of file diff --git a/tools/gate/playbooks/files/hosts.toml b/tools/gate/playbooks/files/hosts.toml new file mode 100644 index 0000000000..4ce2b31497 --- /dev/null +++ b/tools/gate/playbooks/files/hosts.toml @@ -0,0 +1,12 @@ +{% if item.skip_server is not defined or not item.skip_server %} +server = "{{ item.server | default('https://' + item.namespace) }}" +{% endif %} + +[host."{{ item.mirror }}"] +capabilities = ["pull", "resolve", "push"] +{% if item.ca is defined %} +ca = "{{ item.ca }}" +{% endif %} +{% if item.skip_verify %} +skip_verify = true +{% endif %} diff --git a/tools/gate/playbooks/run-scripts.yaml b/tools/gate/playbooks/run-scripts.yaml index 56a2fc0c8a..4f2d5f90b2 100644 --- a/tools/gate/playbooks/run-scripts.yaml +++ b/tools/gate/playbooks/run-scripts.yaml @@ -11,18 +11,48 @@ # limitations under the License. --- -- hosts: all - tasks: - - name: Override images - include_role: - name: override-images - when: buildset_registry is defined - - name: Use docker mirror - include_role: - name: use-docker-mirror - - hosts: primary tasks: + - name: Override images + when: buildset_registry is defined + vars: + work_dir: "{{ zuul.project.src_dir }}" + block: + - name: Buildset registry alias + include_tasks: + file: buildset_registry_alias.yaml + + - name: Print zuul + debug: + var: zuul + + - name: Override proposed images from artifacts + shell: > + find {{ override_paths | join(" ") }} -type f -exec sed -Ei + "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} + + loop: "{{ zuul.artifacts | default([]) }}" + args: + chdir: "{{ work_dir }}" + loop_control: + loop_var: zj_zuul_artifact + when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'" + vars: + tag: "{{ zj_zuul_artifact.metadata.tag }}" + repo: "{{ zj_zuul_artifact.metadata.repository }}" + override_paths: + - ../openstack-helm*/*/values* + - ../openstack-helm-infra/tools/deployment/ + + - name: Diff + shell: | + set -ex; + for dir in openstack-helm openstack-helm-infra; do + path="{{ work_dir }}/../${dir}/" + if [ ! -d "${path}" ]; then continue; fi + echo "${dir} diff" + cd "${path}"; git diff; cd -; + done + - name: "creating directory for run artifacts" file: path: "/tmp/artifacts" diff --git a/zuul.d/jobs-openstack-helm.yaml b/zuul.d/jobs-openstack-helm.yaml index af06d988d6..d84892d232 100644 --- a/zuul.d/jobs-openstack-helm.yaml +++ b/zuul.d/jobs-openstack-helm.yaml @@ -69,6 +69,7 @@ calico_version: "v3.25" helm_version: "v3.6.3" yq_version: "v4.6.0" + crictl_version: "v1.26.1" zuul_osh_infra_relative_path: ../openstack-helm-infra gate_scripts_relative_path: ../openstack-helm