Fix k8s-crio buildset registry test

* It looks like zuul-jobs-test-registry-buildset-registry-k8s-crio
  is busted with Ubuntu Jammy + cri-o installed from kubic, with
  errors like https://github.com/cri-o/ocicni/issues/77
  (also, kubic has been wound down and cri-o has been spun off)
* cri-o in Noble uninstalls docker-ce, in a follow-up we should
  clean that up and switch to a pure podman profile
* This minikube configuration is not supported, but it seems that
  upstream cri-o might have made some fixes that makes it work

* Update the job to use Ubuntu Noble instead of Jammy
* Update ensure-podman for Ubuntu Noble
  (podman is now part of the Ubuntu distro)
* Update the cri-o install in ensure-minikube for Ubuntu Noble and later
  (cri-o is now part of k8s)

Other miscellaneous fixes and workarounds:

* k8s.gcr.io is being sunsetted, updated the test image:
  https://kubernetes.io/blog/2023/03/10/image-registry-redirect/
* Relaxed the security to run minikube from /tmp (in future,
  we should set the default to /usr/local/bin)
* Updated the microk8s check-distro task for Noble

Change-Id: I3b0cbac5c72c31577797ba294de8b8c025f8c2c3
This commit is contained in:
Jan Gutter 2024-08-08 21:04:06 +01:00
parent d8ec17cab0
commit e637029091
No known key found for this signature in database
GPG Key ID: 13F79FC15EC1117C
9 changed files with 144 additions and 50 deletions

View File

@ -0,0 +1,28 @@
- name: Add all repositories
include_role:
name: ensure-package-repositories
vars:
repositories_keys:
- url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/Release.key"
- url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/Release.key"
repositories_list:
- repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /"
- repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/ /"
- name: Install packages
package:
name:
- cri-o
- cri-o-runc
- containernetworking-plugins
- podman
- cri-tools
state: present
become: true
- name: Set crio cgroup driver
ini_file:
path: /etc/crio/crio.conf
section: crio.runtime
option: cgroup_manager
value: '"cgroupfs"'
mode: 0644
become: true

View File

@ -1,28 +1,62 @@
- name: Add all repositories - name: Add all repositories
# Instructions from here: https://github.com/cri-o/packaging making
# the assumption that CRIO_VERSION == KUBERNETES_VERSION
include_role: include_role:
name: ensure-package-repositories name: ensure-package-repositories
vars: vars:
repositories_keys: repositories_keys:
- url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/Release.key" - url: "https://pkgs.k8s.io/core:/stable:/{{ ensure_kubernetes_kubectl_version }}/deb/Release.key"
- url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/Release.key" - url: "https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ ensure_kubernetes_kubectl_version }}/deb/Release.key"
repositories_list: repositories_list:
- repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /" - repo: "deb https://pkgs.k8s.io/core:/stable:/{{ ensure_kubernetes_kubectl_version }}/deb/ /"
- repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.24/xUbuntu_{{ ansible_distribution_version }}/ /" - repo: "deb https://pkgs.k8s.io/addons:/cri-o:/stable:/{{ ensure_kubernetes_kubectl_version }}/deb/ /"
- name: Install packages - name: Install packages
package: package:
name: name:
- cri-o - cri-o
- cri-o-runc - runc
- containernetworking-plugins - containernetworking-plugins
- podman
- cri-tools - cri-tools
- podman
- kubernetes-cni
state: present state: present
become: true become: true
- name: Set crio cgroup driver
# The the following two options are recommended from cri-o install notes
- name: Enable ipv4 forwarding
sysctl:
name: net.ipv4.ip_forward
value: '1'
sysctl_set: true
state: present
reload: true
become: true
- name: Load br_netfilter
modprobe:
name: br_netfilter
state: present
persistent: present
become: true
- name: Find networking plugins
ini_file: ini_file:
path: /etc/crio/crio.conf path: /etc/crio/crio.conf
section: crio.runtime section: crio.network
option: cgroup_manager option: plugin_dirs
value: '"cgroupfs"' value:
- '/opt/cni/bin/'
- '/usr/lib/cni'
mode: 0644 mode: 0644
become: true become: true
register: _crio_conf_updated
# NOTE: want to restart here rather than notify and do it later, so
# that we don't go on without the config correct.
- name: Restart crio to pickup changes # noqa no-handler
service:
name: crio
state: restarted
become: yes
when: _crio_conf_updated.changed

View File

@ -1,7 +1,7 @@
- name: Check distro - name: Check distro
assert: assert:
that: ansible_distribution_release in ['jammy', 'bookworm'] that: ansible_distribution_release in ['jammy', 'bookworm', 'noble']
msg: 'This role only supported on Jammy or Bookworm' msg: 'This role is only supported on Jammy or Bookworm or Noble'
- name: Install snapd - name: Install snapd
become: yes become: yes

View File

@ -3,6 +3,16 @@
path: /tmp/minikube path: /tmp/minikube
register: stat_result register: stat_result
# This is needed because minikube is installed in /tmp
- name: Disable protections for races in /tmp
sysctl:
name: fs.protected_regular
value: '0'
sysctl_set: true
state: present
reload: true
become: true
- name: Download Minikube - name: Download Minikube
get_url: get_url:
url: https://storage.googleapis.com/minikube/releases/{{ minikube_version }}/minikube-linux-amd64 url: https://storage.googleapis.com/minikube/releases/{{ minikube_version }}/minikube-linux-amd64
@ -17,13 +27,28 @@
dest: /usr/local/bin/kubectl dest: /usr/local/bin/kubectl
state: link state: link
- name: Get the kubernetes version
command: >-
/tmp/minikube kubectl --
version --client=true --output=json
changed_when: False
register: ensure_kubernetes_kubectl_version_result
- name: Set the kubernetes version
vars:
kubectl_version: >-
{{ ensure_kubernetes_kubectl_version_result.stdout | from_json }}
set_fact:
ensure_kubernetes_kubectl_version: >-
v{{ kubectl_version['clientVersion']['major'] }}.{{ kubectl_version['clientVersion']['minor'] }}
- name: Run ensure-docker role - name: Run ensure-docker role
include_role: include_role:
name: ensure-docker name: ensure-docker
# Ubuntu focal doesn't have cri-o-1.15 packages, per distro tasks is # Ubuntu doesn't have cri-o packages, per distro tasks is
# required to install crio # required to install cri-o
- name: Install crio - name: Install cri-o
# Note this is required even for the docker runtime, as minikube only # Note this is required even for the docker runtime, as minikube only
# supports cri now. See below for the docker wrapper # supports cri now. See below for the docker wrapper
include_tasks: "{{ zj_distro_os }}" include_tasks: "{{ zj_distro_os }}"

View File

@ -1,12 +1,3 @@
- name: Add kubic project repository
include_role:
name: ensure-package-repositories
vars:
repositories_keys:
- url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/Release.key"
repositories_list:
- repo: "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_distribution_version }}/ /"
- name: Install podman - name: Install podman
package: package:
name: name:
@ -15,21 +6,16 @@
- slirp4netns - slirp4netns
- fuse-overlayfs - fuse-overlayfs
- containernetworking-plugins - containernetworking-plugins
# This enables container network dns resolution:
- golang-github-containernetworking-plugin-dnsname
state: present state: present
become: yes become: yes
# NOTE(pabelanger): Remove default registries.conf file, so we can manage it
# ourself. It could have v1 syntax, which doesn't work with v2.
- name: Remove /etc/containers/registries.conf
become: true
file:
state: absent
path: /etc/containers/registries.conf
- name: Create containers config dir - name: Create containers config dir
file: file:
path: '{{ ansible_user_dir }}/.config/containers' path: '{{ ansible_user_dir }}/.config/containers'
state: directory state: directory
- name: Force cgroup manager to cgroupfs for Ubuntu - name: Force cgroup manager to cgroupfs for Ubuntu
copy: copy:
content: | content: |

View File

@ -2,3 +2,4 @@ buildset_registry_namespaces:
- ['docker.io', 'https://registry-1.docker.io'] - ['docker.io', 'https://registry-1.docker.io']
- ['quay.io', 'https://quay.io'] - ['quay.io', 'https://quay.io']
- ['gcr.io', 'https://gcr.io'] - ['gcr.io', 'https://gcr.io']
- ['registry.k8s.io', 'https://registry.k8s.io']

View File

@ -79,13 +79,38 @@
mode: 0644 mode: 0644
become: true become: true
- name: Restart docker daemon - name: Populate service facts
service: service_facts:
name: docker
state: restarted # This is a copy of the logic from the ensure-docker handlers
become: true - name: Restart docker if it exists
register: docker_restart block:
failed_when: docker_restart is failed and not 'Could not find the requested service' in docker_restart.msg - name: Stop docker.socket to avoid any conflict
become: true
service:
name: docker.socket
enabled: yes
state: stopped
failed_when: false
- name: Assure docker service is running
become: true
service:
name: docker
enabled: yes
state: started
- name: Assure docker.socket service is running
become: true
service:
name: docker.socket
enabled: yes
state: started
failed_when: false
when:
# docker-ce may have been uninstalled by cri-o
- "'docker.service' in ansible_facts.services"
- ansible_facts.services['docker.service']['status'] != 'not-found'
- name: Ensure containers directory exists - name: Ensure containers directory exists
become: yes become: yes

View File

@ -24,7 +24,7 @@
restartPolicy: Never restartPolicy: Never
containers: containers:
- name: test - name: test
image: k8s.gcr.io/pause:3.1 image: registry.k8s.io/pause:3.1
- name: Start pod - name: Start pod
command: kubectl apply -f test-pod.yaml command: kubectl apply -f test-pod.yaml

View File

@ -368,6 +368,11 @@
- test-playbooks/registry/test-registry-post.yaml - test-playbooks/registry/test-registry-post.yaml
vars: vars:
container_command: podman container_command: podman
# There seems to be flakiness in pre-Noble
nodeset:
nodes:
- name: ubuntu-noble
label: ubuntu-noble
- job: - job:
name: zuul-jobs-test-ensure-kubernetes-crio name: zuul-jobs-test-ensure-kubernetes-crio
@ -396,15 +401,6 @@
- name: ubuntu-focal - name: ubuntu-focal
label: ubuntu-focal label: ubuntu-focal
- job:
name: zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
description: Test the ensure-kubernetes role with crio-o on ubuntu-jammy
parent: zuul-jobs-test-ensure-kubernetes-crio
nodeset:
nodes:
- name: ubuntu-jammy
label: ubuntu-jammy
- job: - job:
name: zuul-jobs-test-ensure-kubernetes-microk8s name: zuul-jobs-test-ensure-kubernetes-microk8s
description: | description: |
@ -564,7 +560,6 @@
- zuul-jobs-test-registry-buildset-registry-k8s-microk8s - zuul-jobs-test-registry-buildset-registry-k8s-microk8s
- zuul-jobs-test-registry-buildset-registry-k8s-crio - zuul-jobs-test-registry-buildset-registry-k8s-crio
- zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal - zuul-jobs-test-ensure-kubernetes-crio-ubuntu-focal
- zuul-jobs-test-ensure-kubernetes-crio-ubuntu-jammy
- zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy - zuul-jobs-test-ensure-kubernetes-microk8s-ubuntu-jammy
- zuul-jobs-test-ensure-kubernetes-microk8s-debian-bookworm - zuul-jobs-test-ensure-kubernetes-microk8s-debian-bookworm
- zuul-jobs-test-ensure-skopeo-debian-bookworm - zuul-jobs-test-ensure-skopeo-debian-bookworm