From e86a1c6f962be1fe10bad5f379b2d99f3f1006f9 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 12 Dec 2024 09:32:13 -0800 Subject: [PATCH] Run containers on Noble with docker compose and podman There are two major issues we are trying to address here. The first is that docker-compose (python implementation) is EOL and does not work with python3.12. Instead we need to use docker compose (golang implementation) on newer platforms like Noble. We're taking advantage of the clean break between distro releases to do a migration of the container management system rather than try and replace docker-compose with docker compose in place on existing servers. Second the docker runtime can only deal with mirrors for images hosted on docker hub. This impacts our ability to speculatively test images that are hosted on quay (or elsewhere) with docker since speculative image testing currently relies on mirror configuration to provide unreleased images to test environments. By switching the runtime to podman instead of docker behind docker compose we fix this second problem. Again the clean break between distro releases is a convenient time to make ths witch rather than doing it in place. Some design considerations include: * Not bothering with docker ce packaging and instead relying on packages in Ubuntu Noble * Configuring the podman service to listen on a socket located where docker's socket typically lives. This avoids needing environment overrides every time we run docker compose. * Not adding a special podman group for this. We effectively manage things as root or via sudo so we can keep this simple for now. Future updates may include installation of docker compose and/or podman from upstream sources. We could add a podman group. We may also switch to using user owner podman daemons and reduce some privilege. Change-Id: Ib0a9cdb38b99521bcd7e15c17f6175aea2c042eb --- .../roles/install-docker/files/10-docker.conf | 3 + .../install-docker/files/docker-compose.shim | 9 ++ .../files/docker.socket.override.conf | 13 +++ .../files/podman.socket.override.conf | 13 +++ .../install-docker/tasks/Ubuntu.noble.yaml | 83 +++++++++++++++++++ .../roles/install-docker/tasks/default.yaml | 59 +++++++++++++ .../roles/install-docker/tasks/main.yaml | 72 +++------------- 7 files changed, 192 insertions(+), 60 deletions(-) create mode 100644 playbooks/roles/install-docker/files/docker-compose.shim create mode 100644 playbooks/roles/install-docker/files/docker.socket.override.conf create mode 100644 playbooks/roles/install-docker/files/podman.socket.override.conf create mode 100644 playbooks/roles/install-docker/tasks/Ubuntu.noble.yaml create mode 100644 playbooks/roles/install-docker/tasks/default.yaml diff --git a/playbooks/roles/install-docker/files/10-docker.conf b/playbooks/roles/install-docker/files/10-docker.conf index 165dde567d..eab5a4263d 100644 --- a/playbooks/roles/install-docker/files/10-docker.conf +++ b/playbooks/roles/install-docker/files/10-docker.conf @@ -5,3 +5,6 @@ if $programname startswith 'docker-' then { ?CUSTOM_LOGS stop } + +# TODO(clarkb) If we tag containers with podman- we will +# need new rules in this file. diff --git a/playbooks/roles/install-docker/files/docker-compose.shim b/playbooks/roles/install-docker/files/docker-compose.shim new file mode 100644 index 0000000000..32a782c13d --- /dev/null +++ b/playbooks/roles/install-docker/files/docker-compose.shim @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# The quoting around $@ is important here to try and preserve original quoting +# and pass that through to the underlying command as is. This is particularly +# useful for our docker-compose exec mariadb commands that use a nested shell +# to interpolate database passwords and execute sql queries or db backups. +# +# See "special parameters" in the bash manual for more info. +/usr/bin/docker compose "$@" diff --git a/playbooks/roles/install-docker/files/docker.socket.override.conf b/playbooks/roles/install-docker/files/docker.socket.override.conf new file mode 100644 index 0000000000..d3aa8542f2 --- /dev/null +++ b/playbooks/roles/install-docker/files/docker.socket.override.conf @@ -0,0 +1,13 @@ +[Unit] +Description=Docker Socket for the API + +[Socket] +# Set a socket path that doesn't conflict with the default. +# Podman will be configured to use the default path. +ListenStream=/run/actualdocker.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker + +[Install] +WantedBy=sockets.target diff --git a/playbooks/roles/install-docker/files/podman.socket.override.conf b/playbooks/roles/install-docker/files/podman.socket.override.conf new file mode 100644 index 0000000000..f6cc9d9cc8 --- /dev/null +++ b/playbooks/roles/install-docker/files/podman.socket.override.conf @@ -0,0 +1,13 @@ +[Unit] +Description=Podman API Socket +Documentation=man:podman-system-service(1) + +[Socket] +# Set podman to listen at docker's socket location +# to make docker compose work without environment +# overrides. +ListenStream=/var/run/docker.sock +SocketMode=0660 + +[Install] +WantedBy=sockets.target diff --git a/playbooks/roles/install-docker/tasks/Ubuntu.noble.yaml b/playbooks/roles/install-docker/tasks/Ubuntu.noble.yaml new file mode 100644 index 0000000000..7716960339 --- /dev/null +++ b/playbooks/roles/install-docker/tasks/Ubuntu.noble.yaml @@ -0,0 +1,83 @@ +# We currently only install docker-compose-v2 and podman from the distro +# on Ubuntu Noble. This also pulls in the docker runtime from the distro +# which we're going to not use. This approach may change if the combo +# stops working. + +- name: Install docker-compose-v2 and podman and friends + become: true + apt: + name: + - docker-compose-v2 + - podman + - uidmap + - slirp4netns + - fuse-overlayfs + - containernetworking-plugins + # This enables container network dns resolution: + - golang-github-containernetworking-plugin-dnsname + # TODO do we need these extra tools? + - buildah + - skopeo + state: present + +- name: Disable docker daemon service + # docker-compose-v2 depends on the docker service. Disable it. + become: true + service: + name: docker + state: stopped + enabled: false + +- name: Disable docker socket service + # docker-compose-v2 depends on the docker service. Disable it. + become: true + service: + name: docker.socket + state: stopped + enabled: false + +# We add this config so that if docker starts it doesn't conflict +# with podman. Mostly belts and suspenders here. +- name: Add docker socket override config + become: true + copy: + src: docker.socket.override.conf + dest: /etc/systemd/system/docker.socket + owner: root + group: root + mode: '0644' + +# Configure the podman socket to pretend to be a docker socket +- name: Add podman socket override config + become: true + copy: + src: podman.socket.override.conf + dest: /etc/systemd/system/podman.socket + owner: root + group: root + mode: '0644' + register: write_podman_sock_config + +- name: Reconfigure the podman services + # We use the podman service instead + become: true + systemd_service: + name: podman.socket + daemon_reload: true + state: restarted + enabled: true + when: write_podman_sock_config.changed + +# Currently we assume container management will be performed by root. +# For this reason we don't do any special group management. However, +# if this changes this is a good location to update groups and reset +# the ansible ssh connection. + +- name: Add docker-compose to docker compose shim + become: true + copy: + src: docker-compose.shim + dest: /usr/local/bin/docker-compose + owner: root + group: root + mode: '0755' diff --git a/playbooks/roles/install-docker/tasks/default.yaml b/playbooks/roles/install-docker/tasks/default.yaml new file mode 100644 index 0000000000..bc1de94616 --- /dev/null +++ b/playbooks/roles/install-docker/tasks/default.yaml @@ -0,0 +1,59 @@ +- name: Create docker directory + become: yes + file: + state: directory + path: /etc/docker + +- name: Install docker-ce from upstream + include_tasks: upstream.yaml + when: use_upstream_docker|bool + +- name: Install docker-engine from distro + include_tasks: distro.yaml + when: not use_upstream_docker|bool + +- name: reset ssh connection to pick up docker group + meta: reset_connection + +# We install docker-compose from pypi to get features like +# stop_grace_period. + +# On arm64 we need build-essential, python3-dev, libffi-dev, and +# libssl-dev because wheels don't exist for all the things on arm64. +# Similarly for Xenial while we have it, some things (cffi) have +# stopped providing Python 3.5 wheels +- name: Install arm64 dev pacakges + when: > + ansible_architecture == 'aarch64' or + ansible_distribution_release == 'xenial' + package: + name: + - build-essential + - python3-dev + - libffi-dev + - libssl-dev + state: present + +- name: Install python docker-compose if needed + when: with_python_compose|bool + block: + - name: ensure pip3 is installed + include_role: + name: pip3 + + - name: Install docker-compose + pip: + name: + # The explicit pin of requests is a temporary workaround to getting + # docker-compose functioning again after requests and urllib3 updates. + # Unfortunately python docker-compose is abandonware and we will need + # to migrate to the new docker plugin system or distro packages, but + # until then this is a quick workaround that will get things moving + # again. + # The explicit pin of docker is required as py docker 7.0 introduced + # incompatibilities with python docker-compose. + - requests<2.30.0 + - docker<7.0.0 + - docker-compose + state: present + executable: pip3 diff --git a/playbooks/roles/install-docker/tasks/main.yaml b/playbooks/roles/install-docker/tasks/main.yaml index e340f703a9..381d4aa906 100644 --- a/playbooks/roles/install-docker/tasks/main.yaml +++ b/playbooks/roles/install-docker/tasks/main.yaml @@ -1,62 +1,14 @@ -- name: Create docker directory - become: yes - file: - state: directory - path: /etc/docker - -- name: Install docker-ce from upstream - include_tasks: upstream.yaml - when: use_upstream_docker|bool - -- name: Install docker-engine from distro - include_tasks: distro.yaml - when: not use_upstream_docker|bool - -- name: reset ssh connection to pick up docker group - meta: reset_connection - -# We install docker-compose from pypi to get features like -# stop_grace_period. - -# On arm64 we need build-essential, python3-dev, libffi-dev, and -# libssl-dev because wheels don't exist for all the things on arm64. -# Similarly for Xenial while we have it, some things (cffi) have -# stopped providing Python 3.5 wheels -- name: Install arm64 dev pacakges - when: > - ansible_architecture == 'aarch64' or - ansible_distribution_release == 'xenial' - package: - name: - - build-essential - - python3-dev - - libffi-dev - - libssl-dev - state: present - -- name: Install python docker-compose if needed - when: with_python_compose|bool - block: - - name: ensure pip3 is installed - include_role: - name: pip3 - - - name: Install docker-compose - pip: - name: - # The explicit pin of requests is a temporary workaround to getting - # docker-compose functioning again after requests and urllib3 updates. - # Unfortunately python docker-compose is abandonware and we will need - # to migrate to the new docker plugin system or distro packages, but - # until then this is a quick workaround that will get things moving - # again. - # The explicit pin of docker is required as py docker 7.0 introduced - # incompatibilities with python docker-compose. - - requests<2.30.0 - - docker<7.0.0 - - docker-compose - state: present - executable: pip3 +# We're taking a different approach with Noble and beyond. +# For these newer releases we're going to use `docker compose` +# with the podman service instead of `docker-compose` with the +# docker service. We'll use task file lookups to differentiate. +# TODO(clarkb) the noble behavior really probably deservices to be +# in a new role but to simplify our transition between container +# runtimes we continue to manage it in install-docker. +- name: Include OS-release specific tasks + include_tasks: "{{ lookup('first_found', file_list) }}" + vars: + file_list: "{{ distro_lookup_path }}" - name: Install rsyslog redirector for container tags copy: @@ -79,7 +31,7 @@ group: adm mode: 0775 -- name: Install log rotation for docker files +- name: Install log rotation for container log files include_role: name: logrotate vars: