Test zuul-executor on focal

We want to replace the current executors with focal executors.
Make sure zuul-executor can run there.

Kubic is apparently the new source for libcontainers stuff:

  https://podman.io/getting-started/installation.html

Use only timesyncd on focal

ntp and timesyncd have a hard conflict with each other. Our test
images install ntp. Remove it and just stay with timesyncd.

Change-Id: I0126f7c77d92deb91711f38a19384a9319955cf5

.zuul.yaml

@@ -1588,6 +1588,8 @@
           label: ubuntu-bionic
         - name: zm01.openstack.org
           label: ubuntu-xenial
+        - name: ze01.opendev.org
+          label: ubuntu-focal
         - name: ze01.openstack.org
           label: ubuntu-xenial
         - name: zuul01.openstack.org
@@ -1604,6 +1606,9 @@
       zm01.openstack.org:
         host_copy_output:
           '/etc/zuul/zuul.conf': logs
+      ze01.opendev.org:
+        host_copy_output:
+          '/etc/zuul/zuul.conf': logs
       ze01.openstack.org:
         host_copy_output:
           '/etc/zuul/zuul.conf': logs

inventory/groups.yaml

@@ -257,6 +257,8 @@ groups:
     - zuul[0-9]*.open*.org
   zuul-executor:
     - ze[0-9]*.open*.org
+  zuul-executor-opendev:
+    - ze[0-9]*.opendev.org
   zuul-merger:
     - zm[0-9]*.open*.org
   zuul-preview:

playbooks/group_vars/zuul-executor-opendev.yaml

@@ -0,0 +1,3 @@
+# NOTE: ubuntu focal has new enough docker, and upstream docker
+# does not have focal support yet.
+use_upstream_docker: false

playbooks/group_vars/zuul-executor.yaml

@@ -20,3 +20,25 @@ zuul_connections:
     user: 'git-infra-root.openstack.org'
     stream_events: 'false'
     auth_type: 'basic'
+libcontainers_apt_key: |
+  -----BEGIN PGP PUBLIC KEY BLOCK-----
+  Version: GnuPG v2.0.15 (GNU/Linux)
+
+  mQENBFtkV0cBCADStSTCG5qgYtzmWfymHZqxxhfwfS6fdHJcbGUeXsI5dxjeCWhs
+  XarZm6rWZOd5WfSmpXhbKOyM6Ll+6bpSl5ICHLa6fcpizYWEPa8fpg9EGl0cF12G
+  GgVLnnOZ6NIbsoW0LHt2YN0jn8xKVwyPp7KLHB2paZh+KuURERG406GXY/DgCxUx
+  Ffgdelym/gfmt3DSq6GAQRRGHyucMvPYm53r+jVcKsf2Bp6E1XAfqBrD5r0maaCU
+  Wvd7bi0B2Q0hIX0rfDCBpl4rFqvyaMPgn+Bkl6IW37zCkWIXqf1E5eDm/XzP881s
+  +yAvi+JfDwt7AE+Hd2dSf273o3WUdYJGRwyZABEBAAG0OGRldmVsOmt1YmljIE9C
+  UyBQcm9qZWN0IDxkZXZlbDprdWJpY0BidWlsZC5vcGVuc3VzZS5vcmc+iQE+BBMB
+  CAAoBQJbZFdHAhsDBQkEHrAABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBN
+  ZDkDdQYKpL0BCACEuegX4bxPkdnML70BbvBmDazyuWYZtSAOLoTosVwXBlTeAIoY
+  xAw2Sdc895808blU964fuu8IDcR5KuvkTGSGIDBJ6fHZSFD8r68WSS0k1gOY7bUW
+  8kkeThlvuMATgULJIYGQ3vYrEhn5e+Pe8fb3AJmQpQWzcjtPX+pk66sVUT6iNaSt
+  QtBVGNTQvVDHekF1j/mYmVbIjIWL6whYkdtlu7KYzCx6KhY9u7i3THxY5ICsq64b
+  rYcy5FHaVv5SHfTvtu1WtiSlACVfOgPGntpI+Xi4P2F4n4c5I5VKBxpHteW4LGz8
+  YOISzS7ZZlbZ7qus353OU0v6UbA660ju5/gGiEYEExECAAYFAltkV0cACgkQOzAR
+  t2udZSOoswCdF44NTN09DwhPFbNYhEMb9juP5ykAn0bcELvuKmgDwEwZMrPQkG8t
+  Pu9n
+  =CNmf
+  -----END PGP PUBLIC KEY BLOCK-----

playbooks/roles/base-repos/files/sources.list.focal.x86_64

@@ -0,0 +1,7 @@
+# This file is kept updated by ansible, adapted from
+# https://help.ubuntu.com/lts/serverguide/configuration.html
+
+deb http://us.archive.ubuntu.com/ubuntu focal main universe
+deb http://us.archive.ubuntu.com/ubuntu focal-updates main universe
+deb http://us.archive.ubuntu.com/ubuntu focal-backports main universe
+deb http://security.ubuntu.com/ubuntu focal-security main universe

playbooks/roles/base-server/tasks/Debian.yaml

@@ -26,12 +26,19 @@
         enabled: yes
         state: started
 
-- name: Ensure systemd-timesyncd is running
-  service:
-    name: systemd-timesyncd
-    enabled: yes
-    state: started
+- name: Remove ntp and run timesyncd
   when: ansible_distribution_version is version('18.04', '>=')
+  block:
+    - name: Remove ntp
+      package:
+        name: ntp
+        state: absent
+
+    - name: Ensure systemd-timesyncd is running
+      service:
+        name: systemd-timesyncd
+        enabled: yes
+        state: started
 
 - name: Remove packages that make no sense for our servers
   package:

playbooks/roles/zuul-executor/tasks/main.yaml

@@ -1,17 +1,41 @@
+- name: Include OS-specific variables
+  include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files: "{{ distro_lookup_path }}"
+      paths:
+        - 'vars'
+
 - name: Install PPAs
   apt_repository:
     repo: '{{ item }}'
   become: yes
-  loop:
-    # For bubblewrap
-    - ppa:openstack-ci-core/bubblewrap
-    # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
-    - ppa:openstack-ci-core/python-bpo-27945-backport
-    # We use later HWE kernels for better memory managment, requiring an
-    # updated AFS version which we install from our custom ppa.
-    - ppa:openstack-ci-core/openafs-amd64-hwe
-    # For skopeo
-    - ppa:projectatomic/ppa
+  loop: '{{ zuul_executor_ppas }}'
+
+- name: Atomic for focal
+  when: ansible_distribution_version is version('20.04', '>=')
+  block:
+
+    - name: Add Kubic libcontainers OBS repo key
+      become: yes
+      apt_key:
+        data: "{{ libcontainers_apt_key }}"
+        keyring: /etc/apt/trusted.gpg.d/projectatomic.gpg
+
+    - name: Add kubic project libcontainers apt repo
+      become: yes
+      template:
+        dest: /etc/apt/sources.list.d/projectatomic.list
+        group: root
+        mode: 0644
+        owner: root
+        src: sources.list.j2
+      register: projectatomic_repo
+
+    - name: Run the equivalent of "apt-get update" as a separate step
+      apt:
+        update_cache: yes
+      when: projectatomic_repo is changed
 
 - name: Install bindep
   pip:
@@ -25,11 +49,16 @@
     name: '{{ item }}'
     state: present
   loop:
-    - libjemalloc1
     - bubblewrap
     - skopeo
     - socat
 
+- name: Install distro-specific executor extra packages
+  package:
+    name: '{{ item }}'
+    state: present
+  loop: '{{ zuul_executor_extra_packages }}'
+
 - name: Clone zuul repo
   git:
     repo: https://opendev.org/zuul/zuul
@@ -117,6 +146,11 @@
   vars:
     logrotate_file_name: /var/log/zuul/executor-debug.log
 
+- name: Install defaults file
+  template:
+    src: zuul-executor.defaults.j2
+    dest: /etc/default/zuul-executor
+
 - name: Install init script
   copy:
     src: zuul-executor.init

playbooks/roles/zuul-executor/templates/sources.list.j2

@@ -0,0 +1 @@
+deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_{{ ansible_lsb.release }}/ /

playbooks/roles/zuul-executor/templates/zuul-executor.defaults.j2

@@ -0,0 +1,4 @@
+PIDFILE=/var/run/zuul/executor.pid
+{% if ansible_distribution_version is version('20.04', '<') %}
+export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+{% endif %}

playbooks/roles/zuul-executor/vars/Ubuntu.focal.yaml

@@ -0,0 +1,5 @@
+zuul_executor_ppas:
+  # We use later HWE kernels for better memory managment, requiring an
+  # updated AFS version which we install from our custom ppa.
+  - ppa:openstack-ci-core/openafs
+zuul_executor_extra_packages: []

playbooks/roles/zuul-executor/vars/default.yaml

@@ -0,0 +1,12 @@
+zuul_executor_ppas:
+  # For bubblewrap
+  - ppa:openstack-ci-core/bubblewrap
+  # Temporary PPA needed for bpo-27945 while waiting for SRU to be published
+  - ppa:openstack-ci-core/python-bpo-27945-backport
+  # We use later HWE kernels for better memory managment, requiring an
+  # updated AFS version which we install from our custom ppa.
+  - ppa:openstack-ci-core/openafs
+  # For skopeo
+  - ppa:projectatomic/ppa
+zuul_executor_extra_packages:
+  - libjemalloc1

testinfra/test_base.py

@@ -80,9 +80,9 @@ def test_iptables(host):
 
 def test_ntp(host):
     package = host.package("ntp")
-    assert package.is_installed
+    if host.system_info.codename in ('xenial'):
+        assert package.is_installed
 
-    if host.system_info.codename != 'bionic':
         if host.system_info.distribution in ['ubuntu', 'debian']:
             service = host.service("ntp")
         else:
@@ -91,11 +91,18 @@ def test_ntp(host):
         assert service.is_enabled
 
     else:
+        assert not package.is_installed
+
         service = host.service('systemd-timesyncd')
         assert service.is_running
 
+        # Focal updates the status string to just say NTP
+        if host.system_info.codename == 'bionic':
+            stdout_string = 'systemd-timesyncd.service active'
+        else:
+            stdout_string = 'NTP service: active'
         cmd = host.run("timedatectl status")
-        assert 'systemd-timesyncd.service active: yes' in cmd.stdout
+        assert stdout_string in cmd.stdout
 
 
 def test_snmp(host):