ansible-lint: use matchplay instead of matchtask

For some reason matchtask doesn't match includes, matchplay does so use
that instead.

Change-Id: I040f7f3394503e92d06c05e8ff671a43b14baebc

.rules/ZuulJobsNamespaceLoopVar.py

@@ -15,9 +15,10 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\
 
     tags = {'zuul-jobs-namespace-loop-var'}
 
-    def matchtask(self, file, task):
+    def matchplay(self, file, task):
-        if file.get('type') != 'tasks':
+        results = []
-            return False
+        if file.get('type') not in ('tasks', 'handlers'):
+            return results
 
         has_loop = 'loop' in task
         for key in task.keys():
@@ -26,10 +27,10 @@ https://zuul-ci.org/docs/zuul-jobs/policy.html\
 
         if has_loop:
             if 'loop_control' not in task:
-                return True
+                results.append(("", self.shortdesc))
             elif 'loop_var' not in task.get('loop_control'):
-                return True
+                results.append(("", self.shortdesc))
             elif not task.get('loop_control')\
                     .get('loop_var').startswith('zj_'):
-                return True
+                results.append(("", self.shortdesc))
-        return False
+        return results

roles/build-container-image/tasks/build.yaml

@@ -1,6 +1,6 @@
 - name: Check sibling directory
   stat:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
   register: _dot_zuul_siblings
 
 # This should have been cleaned up; multiple builds may specify
@@ -11,42 +11,42 @@
 
 - name: Create sibling source directory
   file:
-    path: '{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
+    path: '{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
     state: directory
     mode: 0755
-  when: item.siblings is defined
+  when: zj_image.siblings is defined
 
 - name: Copy sibling source directories
   command:
-    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ item.context }}/.zuul-siblings'
+    cmd: 'cp --parents -r {{ zj_sibling }} {{ ansible_user_dir }}/{{ zuul_work_dir }}/{{ zj_image.context }}/.zuul-siblings'
     chdir: '~/src'
-  loop: '{{ item.siblings }}'
+  loop: '{{ zj_image.siblings }}'
   loop_control:
     loop_var: zj_sibling
-  when: item.siblings is defined
+  when: zj_image.siblings is defined
 
 - name: Set container filename arg
   set_fact:
-    containerfile: "{{ item.container_filename|default(container_filename)|default('') }}"
+    containerfile: "{{ zj_image.container_filename|default(container_filename)|default('') }}"
 
 - name: Build a container image
   command: >-
-    {{ container_command }} build {{ item.path | default('.') }} {% if containerfile %}-f {{ containerfile }}{% endif %}
+    {{ container_command }} build {{ zj_image.path | default('.') }} {% if containerfile %}-f {{ containerfile }}{% endif %}
-    {% if item.target | default(false) -%}
+    {% if zj_image.target | default(false) -%}
-      --target {{ item.target }}
+      --target {{ zj_image.target }}
     {% endif -%}
-    {% for build_arg in item.build_args | default([]) -%}
+    {% for build_arg in zj_image.build_args | default([]) -%}
       --build-arg {{ build_arg }}
     {% endfor -%}
-    {% if items.siblings | default(false) -%}
+    {% if zj_images.siblings | default(false) -%}
-      --build-arg "ZUUL_SIBLINGS={{ item.siblings | join(' ') }}"
+      --build-arg "ZUUL_SIBLINGS={{ zj_image.siblings | join(' ') }}"
     {% endif -%}
-    {% for tag in item.tags | default(['latest']) -%}
+    {% for tag in zj_image.tags | default(['latest']) -%}
-    --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }}
+    --tag {{ zj_image.repository }}:change_{{ zuul.change }}_{{ tag }}
-    --tag {{ item.repository }}:{{ tag }}
+    --tag {{ zj_image.repository }}:{{ tag }}
     {% endfor -%}
   args:
-    chdir: "{{ zuul_work_dir }}/{{ item.context }}"
+    chdir: "{{ zuul_work_dir }}/{{ zj_image.context }}"
 
 - name: Cleanup sibling source directory
   file:

roles/build-container-image/tasks/main.yaml

@@ -17,6 +17,8 @@
 - name: Build container images
   include_tasks: build.yaml
   loop: "{{ container_images }}"
+  loop_control:
+    loop_var: zj_image
 
 # Docker, and therefore skopeo and podman, don't understand docker
 # push [1234:5678::]:5000/image/path:tag so we set up /etc/hosts with
@@ -30,18 +32,21 @@
     line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
     insertafter: EOF
   when: buildset_registry is defined and buildset_registry.host | ipaddr
+
 - name: Set buildset_registry alias variable when using ip
   set_fact:
     buildset_registry_alias: zuul-jobs.buildset-registry
   when: buildset_registry is defined and buildset_registry.host | ipaddr
+
 - name: Set buildset_registry alias variable when using name
   set_fact:
     buildset_registry_alias: "{{ buildset_registry.host }}"
   when: buildset_registry is defined and not ( buildset_registry.host | ipaddr )
+
 # Push each image.
 - name: Push image to buildset registry
   when: buildset_registry is defined
   include_tasks: push.yaml
   loop: "{{ container_images }}"
   loop_control:
-    loop_var: image
+    loop_var: zj_image

roles/build-container-image/tasks/push.yaml

@@ -1,12 +1,13 @@
 - name: Tag image for buildset registry
   command: >-
-    {{ container_command }} tag {{ image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
+    {{ container_command }} tag {{ zj_image.repository }}:{{ zj_image_tag }} {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
+
 - name: Push tag to buildset registry
   command: >-
-    {{ container_command }} push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ zj_image_tag }}
+    {{ container_command }} push {{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}
-  loop: "{{ image.tags | default(['latest']) }}"
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag

roles/build-docker-image/tasks/main.yaml

@@ -31,10 +31,12 @@
     line: "{{ buildset_registry.host }}\tzuul-jobs.buildset-registry"
     insertafter: EOF
   when: buildset_registry is defined and buildset_registry.host | ipaddr
+
 - name: Set buildset_registry alias variable when using ip
   set_fact:
     buildset_registry_alias: zuul-jobs.buildset-registry
   when: buildset_registry is defined and buildset_registry.host | ipaddr
+
 - name: Set buildset_registry alias variable when using name
   set_fact:
     buildset_registry_alias: "{{ buildset_registry.host }}"

roles/configure-mirrors/handlers/main.yaml

@@ -9,20 +9,24 @@
 
 - name: Update yum/dnf cache
   become: yes
-  command: "{{ item }}"
+  command: "{{ zj_command }}"
   args:
     warn: false
   with_items:
     - "{{ ansible_pkg_mgr }} clean all"
     - "{{ ansible_pkg_mgr }} makecache -v"
+  loop_control:
+    loop_var: zj_command
   # verbose is needed in order to make it possible to debug potential failures
 
 - name: Update zypper cache
   become: yes
-  command: "{{ item }}"
+  command: "{{ zj_command }}"
   with_items:
     - zypper clean
     - zypper refresh
+  loop_control:
+    loop_var: zj_command
 
 - name: Update Gentoo cache
   become: yes

roles/ensure-bazel/tasks/install-bazel.yaml

@@ -9,7 +9,8 @@
     return_content: true
   register: bazel_installer_checksum
 
-- debug: msg="Checksum is {{ bazel_installer_checksum.content.split(' ')[0] }}"
+- debug:
+    msg: "Checksum is {{ bazel_installer_checksum.content.split(' ')[0] }}"
 
 - name: Download bazel installer
   get_url:
@@ -18,12 +19,16 @@
     mode: 0755
     checksum: "sha256:{{ bazel_installer_checksum.content.split(' ')[0] }}"
 
-- debug: msg="Distribution is {{ ansible_distribution }}"
+- debug:
-- debug: msg="OS family is {{ ansible_os_family }}"
+    msg: "Distribution is {{ ansible_distribution }}"
+- debug:
+    msg: "OS family is {{ ansible_os_family }}"
 
 - name: Install bazel and platform-specific dependencies
-  include: "{{ item }}"
+  include: "{{ zj_distro_os }}"
   with_first_found:
     - "{{ ansible_distribution }}.yaml"
     - "{{ ansible_os_family }}.yaml"
     - "default.yaml"
+  loop_control:
+    loop_var: zj_distro_os

roles/ensure-pip/tasks/main.yaml

@@ -27,7 +27,7 @@
   failed_when: false
 
 - name: Install pip from packages
-  include: "{{ item }}"
+  include: "{{ zj_distro_os }}"
   with_first_found:
     - "{{ ansible_distribution_release }}.yaml"
     - "{{ ansible_distribution }}.yaml"
@@ -36,6 +36,8 @@
   when:
     - ensure_pip_from_packages
     - pip_preinstalled.rc != 0
+  loop_control:
+    loop_var: zj_distro_os
 
 - name: Install pip from source
   include: source.yaml

roles/ensure-podman/tasks/main.yaml

@@ -1,11 +1,13 @@
 - name: Find distribution installation
-  include: "{{ item }}"
+  include: "{{ zj_distro_os }}"
   static: no
   with_first_found:
     - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
     - "{{ ansible_distribution }}.yaml"
     - "{{ ansible_os_family }}.yaml"
     - "default.yaml"
+  loop_control:
+    loop_var: zj_distro_os
 
 - name: Validate podman engine
   when: ensure_podman_validate

roles/ensure-virtualenv/tasks/main.yaml

@@ -7,12 +7,14 @@
   failed_when: false
 
 - name: Install virtualenv package
-  include: "{{ item }}"
+  include: "{{ zj_distro_os }}"
   with_first_found:
     - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
     - "{{ ansible_distribution_release }}.yaml"
     - "{{ ansible_distribution }}.yaml"
     - "{{ ansible_os_family }}.yaml"
     - "default.yaml"
+  loop_control:
+    loop_var: zj_distro_os
   when:
     - virtualenv_preinstalled.rc != 0

roles/persistent-firewall/tasks/main.yaml

@@ -19,10 +19,12 @@
    - skip_ansible_lint
 
 - name: Configure persistent iptables rules
-  include: "{{ item }}"
+  include: "{{ zj_distro_os }}"
   static: no
   with_first_found:
     - "persist/{{ ansible_distribution }}_{{ ansible_distribution_release }}.yaml"
     - "persist/{{ ansible_distribution }}.yaml"
     - "persist/{{ ansible_os_family }}.yaml"
     - "persist/default.yaml"
+  loop_control:
+    loop_var: zj_distro_os

roles/phoronix-combine-results/tasks/fetch-result.yaml

@@ -1,10 +1,10 @@
 ---
 - name: Create result dir
   file:
-    path: "{{ ansible_env.HOME }}/test-results/{{ item.url.rstrip('/')|basename }}"
+    path: "{{ ansible_env.HOME }}/test-results/{{ zj_artifact.url.rstrip('/')|basename }}"
     state: directory
 
 - name: Fetch artifact
   get_url:
-    url: "{{ item.url }}/composite.xml"
+    url: "{{ zj_artifact.url }}/composite.xml"
     dest: "{{ ansible_env.HOME }}/test-results/{{ item.url.rstrip('/')|basename }}/composite.xml"

roles/phoronix-combine-results/tasks/main.yaml

@@ -2,10 +2,12 @@
 - name: Fetch previous result
   include_tasks: fetch-result.yaml
   when:
-    - item.metadata is defined
+    - zj_artifact.metadata is defined
-    - item.metadata.type is defined
+    - zj_artifact.metadata.type is defined
-    - item.metadata.type == "phoronix_result"
+    - zj_artifact.metadata.type == "phoronix_result"
   loop: "{{ zuul.artifacts }}"
+  loop_control:
+    loop_var: zj_artifact
 
 - name: List previous result name
   command: "ls {{ ansible_env.HOME }}/test-results/"

roles/prepare-workspace-openshift/tasks/main.yaml

@@ -1,4 +1,6 @@
 ---
 - include_tasks: rsync.yaml
-  when: item.1.pod is defined
+  when: zj_pod.1.pod is defined
   loop: "{{ openshift_pods.items()|list }}"
+  loop_control:
+    loop_var: zj_pod

roles/prepare-workspace-openshift/tasks/rsync.yaml

@@ -1,17 +1,17 @@
 ---
 - name: Create src directory
   command: >
-    oc --context "{{ item.1.context }}"
+    oc --context "{{ zj_pod.1.context }}"
-       --namespace "{{ item.1.namespace }}"
+       --namespace "{{ zj_pod.1.namespace }}"
-       exec {{ item.1.pod }} mkdir src
+       exec {{ zj_pod.1.pod }} mkdir src
   delegate_to: localhost
 
 - name: Copy src repos to the pod
   command: >
-    oc --context "{{ item.1.context }}"
+    oc --context "{{ zj_pod.1.context }}"
-       --namespace "{{ item.1.namespace }}"
+       --namespace "{{ zj_pod.1.namespace }}"
         rsync -q --progress=false
           {{ zuul.executor.src_root }}/
-          {{ item.1.pod }}:src/
+          {{ zj_pod.1.pod }}:src/
   no_log: true
   delegate_to: localhost

roles/promote-docker-image/tasks/promote-retag-inner.yaml

@@ -1,7 +1,7 @@
 - name: Get manifest
   no_log: true
   uri:
-    url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/change_{{ zuul.change }}_{{ image_tag }}"
+    url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/change_{{ zuul.change }}_{{ zj_image_tag }}"
     status_code: 200
     headers:
       Accept: "application/vnd.docker.distribution.manifest.v2+json"
@@ -11,7 +11,7 @@
 - name: Put manifest
   no_log: true
   uri:
-    url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/{{ image_tag }}"
+    url: "https://registry.hub.docker.com/v2/{{ zj_image.repository }}/manifests/{{ zj_image_tag }}"
     method: PUT
     status_code: 201
     body: "{{ manifest.content | string }}"
@@ -21,7 +21,7 @@
 - name: Delete the current change tag
   no_log: true
   uri:
-    url: "https://hub.docker.com/v2/repositories/{{ zj_image.repository }}/tags/change_{{ zuul.change }}_{{ image_tag }}/"
+    url: "https://hub.docker.com/v2/repositories/{{ zj_image.repository }}/tags/change_{{ zuul.change }}_{{ zj_image_tag }}/"
     method: DELETE
     status_code: [200, 204]
     headers:

roles/promote-docker-image/tasks/promote-retag.yaml

@@ -6,8 +6,9 @@
     password: "{{ docker_credentials.password | trim }}"
     force_basic_auth: true
   register: token
+
 - name: Retag image
   loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
-    loop_var: image_tag
+    loop_var: zj_image_tag
   include_tasks: promote-retag-inner.yaml

roles/stage-output/tasks/main.yaml

@@ -89,8 +89,10 @@
   register: log_files_to_rename
 
 - name: Rename log files that match extension_list
-  shell: "mv {{ item.path }} {{ item.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}"
+  shell: "mv {{ zj_log_file.path }} {{ zj_log_file.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}"
   with_items: "{{ log_files_to_rename.files }}"
+  loop_control:
+    loop_var: zj_log_file
   args:
     chdir: "{{ stage_dir }}/logs"
   tags:

roles/upload-docker-image/tasks/main.yaml

@@ -7,11 +7,13 @@
     loop_var: zj_image
   fail:
     msg: "{{ zj_image.repository }} not permitted by {{ docker_credentials.repository }}"
+
 - name: Log in to dockerhub
   command: "docker login -u {{ docker_credentials.username }} -p {{ docker_credentials.password }}"
   no_log: true
+
 - name: Upload image to dockerhub
   loop: "{{ docker_images }}"
   loop_control:
-    loop_var: image
+    loop_var: zj_image
   include_tasks: push.yaml

roles/upload-docker-image/tasks/push.yaml

@@ -1,6 +1,6 @@
 - name: Upload tag to dockerhub
-  command: "docker push {{ image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}"
+  command: "docker push {{ zj_image.repository }}:change_{{ zuul.change }}_{{ zj_image_tag }}"
-  loop: "{{ image.tags | default(['latest']) }}"
+  loop: "{{ zj_image.tags | default(['latest']) }}"
   loop_control:
     loop_var: zj_image_tag
   register: result

test-playbooks/ansible-lint-rules/roles/handlers-loopvar-loop/handlers/main.yaml

@@ -0,0 +1,8 @@
+- debug:
+    msg: "I should pass: {{ zj_item }}"
+  loop:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item
+

test-playbooks/ansible-lint-rules/roles/handlers-loopvar-with/handlers/main.yaml

@@ -0,0 +1,8 @@
+- debug:
+    msg: "I should pass: {{ zj_item }}"
+  with_list:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item
+

test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-loop/handlers/main.yaml

@@ -0,0 +1,5 @@
+- debug:
+    msg: "I should fail: {{ item }}"
+  loop:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/roles/handlers-missing-loopvar-with/handlers/main.yaml

@@ -0,0 +1,5 @@
+- debug:
+    msg: "I should fail: {{ item }}"
+  with_list:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-loop/tasks/main.yaml

@@ -0,0 +1,7 @@
+- include: "{{ zj_item }}.yaml"
+  loop:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item
+

test-playbooks/ansible-lint-rules/roles/tasks-include-loopvar-with/tasks/main.yaml

@@ -0,0 +1,6 @@
+- include: "{{ zj_item }}.yaml"
+  with_list:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item

test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-loop/tasks/main.yaml

@@ -0,0 +1,4 @@
+- include: "{{ item }}.yaml"
+  loop:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/roles/tasks-include-missing-loopvar-with/tasks/main.yaml

@@ -0,0 +1,4 @@
+- include: "{{ item }}.yaml"
+  with_list:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/roles/tasks-loopvar-loop/tasks/main.yaml

@@ -0,0 +1,8 @@
+- debug:
+    msg: "I should pass: {{ zj_item }} "
+  loop:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item
+

test-playbooks/ansible-lint-rules/roles/tasks-loopvar-with/tasks/main.yaml

@@ -0,0 +1,7 @@
+- debug:
+    msg: "I should pass: {{ zj_item }} "
+  with_list:
+    - 1
+    - 2
+  loop_control:
+    loop_var: zj_item

test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-loop/tasks/main.yaml

@@ -0,0 +1,5 @@
+- debug:
+    msg: "I should error: {{ item }} "
+  loop:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/roles/tasks-missing-loopvar-with/tasks/main.yaml

@@ -0,0 +1,5 @@
+- debug:
+    msg: "I should error: {{ item }} "
+  with_list:
+    - 1
+    - 2

test-playbooks/ansible-lint-rules/run.yaml

@@ -0,0 +1,42 @@
+- hosts: all
+  roles:
+    - ensure-pip
+  tasks:
+    - name: Create tempdir for ansible-lint venv
+      tempfile:
+        state: directory
+      register: ansible_lint_tempdir
+
+    - name: Install test-requirements.txt
+      pip:
+        requirements: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}/test-requirements.txt"
+        virtualenv: "{{ ansible_lint_tempdir.path }}"
+
+    - name: Make sure ansible-lint is installed
+      command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint --version"
+
+    - name: Make sure missing loopvars fail linting
+      command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
+      args:
+        chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
+      register: ansible_lint
+      failed_when: ansible_lint.rc == 0
+      loop:
+        - tasks-missing-loopvar-with
+        - tasks-missing-loopvar-loop
+        - tasks-include-missing-loopvar-with
+        - tasks-include-missing-loopvar-loop
+        - handlers-missing-loopvar-with
+        - handlers-missing-loopvar-loop
+
+    - name: Make sure valid configuration passes linting
+      command: "{{ ansible_lint_tempdir.path }}/bin/ansible-lint test-playbooks/ansible-lint-rules/roles/{{ item }}"
+      args:
+        chdir: "{{ansible_user_dir}}/{{ zuul.project.src_dir }}"
+      loop:
+        - tasks-loopvar-with
+        - tasks-loopvar-loop
+        - tasks-include-loopvar-with
+        - tasks-include-loopvar-loop
+        - handlers-loopvar-with
+        - handlers-loopvar-loop

zuul-tests.d/ansible-list-rules.yaml

@@ -0,0 +1,16 @@
+- job:
+    name: zuul-jobs-test-ansible-lint-rules
+    description: |
+      Test custom ansible-lint rules in zuul-jobs
+    run: test-playbooks/ansible-lint-rules/run.yaml
+    files:
+      - ^/.rules/.*
+      - ^test-playbooks/ansible-lint-rules/.*
+      - ^\.ansible-lint
+
+- project:
+    check:
+      jobs: &id001
+        - zuul-jobs-test-ansible-lint-rules
+    gate:
+      jobs: *id001