Fixup CI jobs

Several things have bit rotted in here that we need to take care of.

First is that we updated the default nodeset to Noble which breaks our
ability to install Pillow<10 for blockdiag. To fix this we need to
install libjpeg-dev so that we can build a Pillow wheel locally during
testing.

Next old ansible-lint doesn't run on Noble's python3.12. We bump up
Ansible lint to a modern version that matches Zuul's current default
Ansible. We also stop installing zuul to get zuul_console and
zuul_return and instead simply mock them in the linter. To make this
work we have to drop the ansible-playbook syntax check run which is fine
because ansible-lint runs this too, but when done via ansible-lint the
mocked modules are respected [0].

Finally we have to clean up/ignore some of the new linter
warnings/errors.

[0] https://ansible.readthedocs.io/projects/lint/rules/syntax-check/

Change-Id: Ia0e936fefc9e2b0f2fa614c93a2f168e14b2825b

.ansible-lint

@@ -8,3 +8,9 @@ skip_list:
   - '204'  # Lines should be no longer than 160 chars
   - '301'  # Commands should not change things if nothing needs doing
   - '701'  # No 'galaxy_info' found
+  - 'fqcn[action-core]'  # We find the shorter names easier to read
+  - 'fqcn[action]'  # We find the shorter names easier to read
+  - 'name[play]'  # Name your plays but they dont all need names
+mock_modules:
+  - zuul_console
+  - zuul_return

bindep.txt

@@ -3,6 +3,8 @@
 
 libffi-devel [test platform:rpm]
 libffi-dev [test platform:dpkg]
+# For Building Pillow wheels
+libjpeg-dev [test doc platform:dpkg]
 libssl-dev [platform:dpkg]
 openssl-devel [platform:rpm]
 python3-dev [compile test platform:dpkg platform:apk]

playbooks/artifacts/promote-deployment.yaml

@@ -9,7 +9,7 @@
     - name: Create artifact staging directory
       file:
         state: directory
-        mode: 0755
+        mode: "0755"
         path: "{{ zuul.executor.work_root }}/artifacts"
     - name: Download artifacts
       include_role:
@@ -35,12 +35,12 @@
       file:
         path: "{{ src_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Ensure target directory exists
       file:
         path: "{{ target_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Unpack into publish dir
       # TODO(clarkb) what is the proper way to set mode here?
       unarchive:  # noqa 208

playbooks/artifacts/promote.yaml

@@ -11,7 +11,7 @@
     - name: Create artifact staging directory
       file:
         state: directory
-        mode: 0755
+        mode: "0755"
         path: "{{ zuul.executor.work_root }}/artifacts"
     - name: Download artifacts
       include_role:
@@ -43,7 +43,7 @@
       file:
         path: "{{ target_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Upload to AFS
       command: "cp {{ item.path }} {{ target_dir }}/{{ item.path | basename | regex_replace(name_replacement, name_target) }}"
       when: "item.path is match(name_replacement)"

playbooks/base-minimal/cleanup.yaml

@@ -1,6 +1,6 @@
 - hosts: all
   # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail
   # successful jobs.
-  ignore_errors: yes
+  ignore_errors: true
   roles:
     - remove-build-sshkey

playbooks/base-minimal/post.yaml

@@ -1,4 +1,5 @@
-- hosts: all
+- name: Finalize build
+  hosts: all
   roles:
     - fetch-output
     - merge-output-to-logs
@@ -6,18 +7,18 @@
     - name: Gather debug info on job fail
       when: not (zuul_success | bool)
       block:
-        - name: get df disk usage
+        - name: Get df disk usage
           raw: timeout -k 5 90 df
           failed_when: false
 
-        - name: get df inode usage
+        - name: Get df inode usage
           raw: timeout -k 5 90 df -i
           failed_when: false
 
-        - name: get l2 networking
+        - name: Get l2 networking
           raw: timeout -k 5 90 ip link
           failed_when: false
 
-        - name: get l3 networking
+        - name: Get l3 networking
           raw: timeout -k 5 90 ip addr
           failed_when: false

playbooks/base-test/cleanup.yaml

@@ -1,6 +1,6 @@
 - hosts: all
   # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail
   # successful jobs.
-  ignore_errors: yes
+  ignore_errors: true
   roles:
     - remove-build-sshkey

playbooks/base-test/post-logs.yaml

@@ -1,4 +1,5 @@
-- hosts: localhost
+- name: Upload build logs to swift
+  hosts: localhost
   tasks:
     - name: Include Zuul manifest role
       include_role:

playbooks/base-test/post.yaml

@@ -1,4 +1,5 @@
-- hosts: all
+- name: Finalize build
+  hosts: all
   roles:
     - fetch-output
     - merge-output-to-logs
@@ -6,18 +7,18 @@
     - name: Gather debug info on job fail
       when: not (zuul_success | bool)
       block:
-        - name: get df disk usage
+        - name: Get df disk usage
           raw: timeout -k 5 90 df
           failed_when: false
 
-        - name: get df inode usage
+        - name: Get df inode usage
           raw: timeout -k 5 90 df -i
           failed_when: false
 
-        - name: get l2 networking
+        - name: Get l2 networking
           raw: timeout -k 5 90 ip link
           failed_when: false
 
-        - name: get l3 networking
+        - name: Get l3 networking
           raw: timeout -k 5 90 ip addr
           failed_when: false

playbooks/base-test/pre.yaml

@@ -1,12 +1,12 @@
-- hosts: localhost
+- name: Create job header and inventory
-  name: Create job header and inventory
+  hosts: localhost
   roles:
     - role: emit-job-header
       zuul_log_path_shard_build: true
     - log-inventory
 
-- hosts: all
+- name: Setup Zuul environment
-  name: Setup Zuul environment
+  hosts: all
   pre_tasks:
     # NOTE(pabelanger): Until we hit the validate-host role, we have a minimal
     # set of ansible variables collected by zuul-executor. This doesn't include
@@ -21,8 +21,8 @@
     - start-zuul-console
     - ensure-output-dirs
 
-- hosts: all
+- name: Configure unbound
-  name: Configure unbound
+  hosts: all
   roles:
     # NOTE(pabelanger): We run this role in its own play to ensure unbound is
     # restarted before proceeding with any other role. This is because we use
@@ -30,15 +30,15 @@
     # actions are triggered at the end of each block of tasks in a play.
     - configure-unbound
 
-- hosts: all
+- name: Prepare workspace and configure mirrors
-  name: Prepare workspace and configure mirrors
+  hosts: all
   roles:
     - validate-host
     - test-prepare-workspace-git
     - mirror-info
     - role: configure-mirrors
-      set_apt_mirrors_trusted: True
+      set_apt_mirrors_trusted: true
-      mirror_use_ssl: True
+      mirror_use_ssl: true
       configure_mirrors_components_9_stream:
         'baseos': true
         'baseos-debug': false

playbooks/base/cleanup.yaml

@@ -1,6 +1,6 @@
 - hosts: all
   # NOTE(pabelanger): We ignore_errors for the following tasks as not to fail
   # successful jobs.
-  ignore_errors: yes
+  ignore_errors: true
   roles:
     - remove-build-sshkey

playbooks/base/post.yaml

@@ -6,18 +6,18 @@
     - name: Gather debug info on job fail
       when: not (zuul_success | bool)
       block:
-        - name: get df disk usage
+        - name: Get df disk usage
           raw: timeout -k 5 90 df
           failed_when: false
 
-        - name: get df inode usage
+        - name: Get df inode usage
           raw: timeout -k 5 90 df -i
           failed_when: false
 
-        - name: get l2 networking
+        - name: Get l2 networking
           raw: timeout -k 5 90 ip link
           failed_when: false
 
-        - name: get l3 networking
+        - name: Get l3 networking
           raw: timeout -k 5 90 ip addr
           failed_when: false

playbooks/base/pre.yaml

@@ -1,12 +1,12 @@
-- hosts: localhost
+- name: Create job header and inventory
-  name: Create job header and inventory
+  hosts: localhost
   roles:
     - role: emit-job-header
       zuul_log_path_shard_build: true
     - log-inventory
 
-- hosts: all
+- name: Setup Zuul environment
-  name: Setup Zuul environment
+  hosts: all
   pre_tasks:
     # NOTE(pabelanger): Until we hit the validate-host role, we have a minimal
     # set of ansible variables collected by zuul-executor. This doesn't include
@@ -21,8 +21,8 @@
     - start-zuul-console
     - ensure-output-dirs
 
-- hosts: all
+- name: Configure unbound
-  name: Configure unbound
+  hosts: all
   roles:
     # NOTE(pabelanger): We run this role in its own play to ensure unbound is
     # restarted before proceeding with any other role. This is because we use
@@ -30,15 +30,15 @@
     # actions are triggered at the end of each block of tasks in a play.
     - configure-unbound
 
-- hosts: all
+- name: Prepare workspace and configure mirrors
-  name: Prepare workspace and configure mirrors
+  hosts: all
   roles:
     - validate-host
     - prepare-workspace-git
     - mirror-info
     - role: configure-mirrors
-      set_apt_mirrors_trusted: True
+      set_apt_mirrors_trusted: true
-      mirror_use_ssl: True
+      mirror_use_ssl: true
       configure_mirrors_components_9_stream:
         'baseos': true
         'baseos-debug': false

playbooks/buildset-registry/post.yaml

@@ -13,7 +13,7 @@
       file:
         path: "{{ ansible_user_dir }}/zuul-output/logs/docker"
         state: directory
-        mode: 0755
+        mode: "0755"
 
     - name: Save registry container logs
       when: "'buildset_registry' in docker_ps.stdout"
@@ -26,4 +26,4 @@
     - name: Trigger failure if required
       fail:
         msg: 'Triggering failure for debugging'
-      when: buildset_registry_debug_fail|default(false)|bool
+      when: buildset_registry_debug_fail | default(false) | bool

playbooks/docs/promote.yaml

@@ -18,7 +18,7 @@
       file:
         path: "{{ zuul.executor.work_root }}/docs"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Extract docs archive
       # TODO(clarkb) what is the proper way to set mode on this task?
       vars:
@@ -70,7 +70,7 @@
       file:
         path: "{{ target_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Set redirect target directory
       when: "target_dict.redirect is defined"
       set_fact:
@@ -84,7 +84,7 @@
       copy:
         dest: "{{ redirect_target_dir }}"
         content: "{{ redirect_content }}"
-        mode: 0644
+        mode: "0644"
     - name: Upload to AFS
       include_role:
         name: upload-afs-roots

playbooks/infra-prod/setup-keys.yaml

@@ -21,5 +21,4 @@
     - name: Add bridge.o.o hostkey to known hosts
       known_hosts:
         name: bridge01.opendev.org
-        key: "bridge01.opendev.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2zgQQ0lpnjtST1oINVPo9jTDI0W3IJA+ZSUTsOo6gGzz6M8PN+jyxQ3EQy8H/xPjm2C55yGfaLNzj557P5arI0TkdeNHt4hzRcmpsNUVE4fOtzMblThRSA1ipfhd/DqMUKz1Ofie3hojnhE1ecB86IcdbF9rfJxl0DOuoSQfClJw3kDx5nK+0Ps5lVnSBZ+NUKbnUFizWVjKz6qIe3tZpuHAvuD/S8tM/gJ+3fC7LcssGb+njo5ghhSUl8wasGBXZQuby33mtuX+UBON+h95odanZs3iGMZScPxgchGb4xB3OTyYn0dFG3mnwPuUnrEKluJ/eFXCkM+Q35DMCa+mynco0stt136e2qTrX1jhtcaDerufb7hs2/7zM/q4zbRYKalUhnh0CZshQ3Y1AuwI3ssmwczQpwgTJnceBmmh5xYPDVCGPpAamTPKhl4VSqlfm2Nc56+dcUm0Y6jiQjBhLIbWevq6RWse6K4R39ovPlTN3z037oJxQHsIETrXxHp0="
+        key: "bridge01.opendev.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2zgQQ0lpnjtST1oINVPo9jTDI0W3IJA+ZSUTsOo6gGzz6M8PN+jyxQ3EQy8H/xPjm2C55yGfaLNzj557P5arI0TkdeNHt4hzRcmpsNUVE4fOtzMblThRSA1ipfhd/DqMUKz1Ofie3hojnhE1ecB86IcdbF9rfJxl0DOuoSQfClJw3kDx5nK+0Ps5lVnSBZ+NUKbnUFizWVjKz6qIe3tZpuHAvuD/S8tM/gJ+3fC7LcssGb+njo5ghhSUl8wasGBXZQuby33mtuX+UBON+h95odanZs3iGMZScPxgchGb4xB3OTyYn0dFG3mnwPuUnrEKluJ/eFXCkM+Q35DMCa+mynco0stt136e2qTrX1jhtcaDerufb7hs2/7zM/q4zbRYKalUhnh0CZshQ3Y1AuwI3ssmwczQpwgTJnceBmmh5xYPDVCGPpAamTPKhl4VSqlfm2Nc56+dcUm0Y6jiQjBhLIbWevq6RWse6K4R39ovPlTN3z037oJxQHsIETrXxHp0="  # noqa: yaml[line-length]
-

playbooks/infra-prod/setup-src.yaml

@@ -32,16 +32,16 @@
     # to master.
     - name: Should we run from master
       set_fact:
-        infra_prod_run_from_master: "{{ zuul.pipeline|default('') in ['periodic', 'opendev-prod-hourly'] }}"
+        infra_prod_run_from_master: "{{ zuul.pipeline | default('') in ['periodic', 'opendev-prod-hourly'] }}"
 
     - name: Reset checkouts to master for periodic jobs
-      when: infra_prod_run_from_master|bool
+      when: infra_prod_run_from_master | bool
       block:
         - name: Update system-config from master
           git:
             repo: 'https://opendev.org/opendev/system-config'
             dest: '/home/zuul/src/opendev.org/opendev/system-config'
-            force: yes
+            force: true
             version: 'master'
 
         - name: Update project-config from master
@@ -49,5 +49,5 @@
           git:
             repo: 'https://opendev.org/openstack/project-config'
             dest: '/home/zuul/src/opendev.org/openstack/project-config'
-            force: yes
+            force: true
             version: 'master'

playbooks/nox-docs/publish.yaml

@@ -4,7 +4,8 @@
       when: "zuul.tag is not defined"
       fail:
         msg: "This playbook must be run in a tag-based pipeline (e.g., 'release')."
-    - include_role:
+    - name: Write root marker
+      include_role:
         name: write-root-marker
       vars:
         root_marker_dir: "{{ zuul.executor.log_root }}/docs"
@@ -25,7 +26,7 @@
       file:
         path: "{{ target_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Upload to AFS
       include_role:
         name: upload-afs-roots

playbooks/tox-docs/publish.yaml

@@ -4,7 +4,8 @@
       when: "zuul.tag is not defined"
       fail:
         msg: "This playbook must be run in a tag-based pipeline (e.g., 'release')."
-    - include_role:
+    - name: Write root marker
+      include_role:
         name: write-root-marker
       vars:
         root_marker_dir: "{{ zuul.executor.log_root }}/docs"
@@ -25,7 +26,7 @@
       file:
         path: "{{ target_dir }}"
         state: directory
-        mode: 0755
+        mode: "0755"
     - name: Upload to AFS
       include_role:
         name: upload-afs-roots

playbooks/upload-git-mirror/run.yaml

@@ -1,4 +1,5 @@
 - hosts: localhost
   roles:
-    - name: upload-git-mirror
+    - role: upload-git-mirror
-      git_mirror_repository: "{{ git_mirror_credentials.target_repository.format(zuul=zuul) }}"
+      vars:
+        git_mirror_repository: "{{ git_mirror_credentials.target_repository.format(zuul=zuul) }}"

roles/configure-unbound/handlers/main.yaml

@@ -1,5 +1,5 @@
 - name: Restart unbound
-  become: yes
+  become: true
   service:
     name: unbound
     state: restarted

roles/configure-unbound/tasks/main.yaml

@@ -19,7 +19,7 @@
     - hostvars[inventory_hostname]['ansible_default_ipv6'] is defined
     - hostvars[inventory_hostname]['ansible_default_ipv6']['address'] is defined
   set_fact:
-    unbound_use_ipv6: True
+    unbound_use_ipv6: true
 
 # Use *only* ipv6 resolvers if ipv6 is present and routable.  This
 # avoids traversing potential NAT when using ipv4 which can be
@@ -48,42 +48,42 @@
     - "default.yaml"
 
 - name: Ensure Unbound conf.d directory exists
-  become: yes
+  become: true
   file:
     path: "{{ unbound_confd }}"
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: "0755"
 
 # TODO: Move this to /etc/unbound/conf.d ?
 - name: Configure unbound forwarding
-  become: yes
+  become: true
   template:
     dest: /etc/unbound/forwarding.conf
     owner: root
     group: root
-    mode: 0644
+    mode: "0644"
     src: forwarding.conf.j2
   register: forwarding_config
   notify:
     - Restart unbound
 
 - name: Configure unbound TTL
-  become: yes
+  become: true
   template:
     dest: "{{ unbound_confd }}/ttl.conf"
     owner: root
     group: root
-    mode: 0644
+    mode: "0644"
     src: ttl.conf.j2
   register: ttl_config
   notify:
     - Restart unbound
 
 - name: Start unbound
-  become: yes
+  become: true
   service:
     name: unbound
     state: started
-    enabled: yes
+    enabled: true

roles/mirror-info/tasks/main.yaml

@@ -1,17 +1,17 @@
 - name: Create /etc/ci
-  become: yes
+  become: true
   file:
     path: /etc/ci
     state: directory
     owner: root
     group: root
-    mode: 0755
+    mode: '0755'
 
 - name: Install ci_mirror script
-  become: yes
+  become: true
   template:
     dest: '/etc/ci/mirror_info.sh'
     owner: root
     group: root
-    mode: 0644
+    mode: '0644'
     src: mirror_info.sh.j2

roles/virtualenv-config/tasks/main.yaml

@@ -6,7 +6,7 @@
   file:
     path: '{{ virtualenv_config_file | dirname }}'
     state: directory
-    mode: 0755
+    mode: '0755'
 
 # NOTE(ianw) : 2020-03-27 workaround 46.1.1 broken setuptools
 #  https://github.com/pypa/virtualenv/issues/1752

roles/write-root-marker/tasks/main.yaml

@@ -3,10 +3,10 @@
   copy:
     dest: "{{ root_marker_dir }}/.root-marker"
     content: "Project: {{ zuul.project.name }} Branch: {{ zuul.branch }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}"
-    mode: 0644
+    mode: "0644"
 - name: Write root marker if zuul.tag
   when: "zuul.tag is defined"
   copy:
     dest: "{{ root_marker_dir }}/.root-marker"
     content: "Project: {{ zuul.project.name }} Tag: {{ zuul.tag }} Build: {{ zuul.build }} Revision: {{ zuul.ref }}"
-    mode: 0644
+    mode: "0644"

test-requirements.txt

@@ -2,8 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 flake8
-zuul
-
 # We need to pin the ansible version directly here; per the
 # deprecation policy it should trail the version used by Zuul by 4
 # weeks to give people time to update before these roles start
@@ -11,9 +9,8 @@ zuul
 # here to pull in ansible anyway; pip doesn't actually have a
 # dependency solver and the uncapped ansible requirement from
 # ansible-lint pull in the latest version.
-ansible>=2.9,<2.10  # <-- keep it as old as possible
+ansible>=8,<9
-# We need to pin ansible-lint to before 4.0 which blows up all over the place
+ansible-lint<25.0.0
-ansible-lint>=5.0.0,<6
 bashate>=0.2
 zuul-sphinx>=0.1.1
 stestr>=1.0.0 # Apache-2.0

tox.ini

@@ -19,8 +19,6 @@ commands =
 
 [testenv:linters]
 setenv =
-  ANSIBLE_LIBRARY= {envsitepackagesdir}/zuul/ansible/base/library
-  ANSIBLE_ACTION_PLUGINS = {envsitepackagesdir}/zuul/ansible/base/action
   # NOTE(pabelanger): if you'd like to run tox -elinters locally,
   # you'll need to export ANSIBLE_ROLES_PATH pointing to the required
   # repos.
@@ -40,10 +38,6 @@ commands =
   {toxinidir}/tools/check_jobs_documented.py
   # Ansible lint
   ansible-lint -v
-  # Ansible Syntax Check
-  bash -c "find playbooks -type f -regex '.*.ya?ml' -exec \
-    ansible-playbook --syntax-check -i {toxinidir}/tests/inventory \
-    \{\} + > /dev/null"
 
 [testenv:venv]
 commands = {posargs}