Switch legacy Zuul jobs to native Zuul v3 jobs

See
https://governance.openstack.org/tc/goals/selected/victoria/native-zuulv3-jobs.html
for more details.

Change-Id: I7b9251caecc0ef5f4c07814ef17c7a025631ab85

.zuul.yaml

@@ -1,13 +1,15 @@
 - job:
-    name: validations-dsvm-functional
+    name: validations-libs-functional
-    parent: legacy-dsvm-base
+    parent: devstack
-    run: playbooks/legacy/validations-dsvm-functional/run.yaml
+    run: playbooks/validations-libs-functional.yaml
-    post-run: playbooks/legacy/validations-dsvm-functional/post.yaml
+    post-run: playbooks/post.yaml
     timeout: 7200
     required-projects:
-      - openstack/devstack-gate
       - openstack/validations-libs
       - openstack/validations-common
+    vars:
+      devstack_localrc:
+        USE_PYTHON3: true
     irrelevant-files:
       - ^.*\.rst$
       - ^doc/.*$
@@ -20,8 +22,8 @@
       - openstack-python3-victoria-jobs
     check:
       jobs:
-        - validations-dsvm-functional
+        - validations-libs-functional
     gate:
       jobs:
         - openstack-tox-linters
-        - validations-dsvm-functional
+        - validations-libs-functional

playbooks/legacy/validations-dsvm-functional/post.yaml

@@ -1,96 +0,0 @@
-- hosts: primary
-  tasks:
-    - name: Set fact for validation src directory
-      set_fact:
-        src_dir: '/home/zuul/src/opendev.org/openstack'
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*nose_results.html
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testr_results.html.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.testrepository/tmp*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*testrepository.subunit.gz
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}/tox'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/.tox/*/log/*
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
-
-    - name: Copy files from {{ src_dir }} on node
-      synchronize:
-        src: '{{ src_dir }}'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=**/*.log
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs

playbooks/legacy/validations-dsvm-functional/run.yaml

@@ -1,37 +0,0 @@
----
-- hosts: primary
-  name: validations-dsvm-functional
-  tasks:
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - name: Set fact for validation src directory
-      set_fact:
-        src_dir: '/home/zuul/src/opendev.org/openstack'
-
-    - name: Set fact for validation executable
-      set_fact:
-        val_exec: "python {{ src_dir }}/validations-common/validations_common/validation.py"
-
-    - include: setup.yaml src_dir="{{ src_dir }}" log_dir="/var/log/validations"
-
-    - include: validations.yaml validation_command="{{ item.command }}" val_output="{{ item.output }}" src_dir="{{ src_dir }}"
-      with_items:
-        - { output: "{{ src_dir }}/run.log", command: "{{ val_exec }} run --validation  check-ftype,512e \
-           --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \
-           --ansible-base-dir {{ src_dir }}/validations-common/validations_common \
-           --output-log {{ src_dir }}/run.log" }
-        - { output: "{{ src_dir }}/run-group.log", command: "{{ val_exec }} run --group prep \
-           --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \
-           --ansible-base-dir {{ src_dir }}/validations-common/validations_common \
-           --output-log {{ src_dir }}/run-group.log" }
-        - { output: "{{ src_dir }}/list.log", command: "{{ val_exec }} list \
-           --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \
-           --ansible-base-dir {{ src_dir }}/validations-common/validations_common \
-           --output-log {{ src_dir }}/list.log" }
-        - { output: "{{ src_dir }}/show.log", command: "{{ val_exec }} show \
-           --validation-dir {{ src_dir }}/validations-common/validations_common/playbooks \
-           --ansible-base-dir {{ src_dir }}/validations-common/validations_common \
-           --output-log {{ src_dir }}/show.log" }

playbooks/legacy/validations-dsvm-functional/setup.yaml

@@ -1,60 +0,0 @@
-- name: Instal python pip
-  ignore_errors: yes
-  become: yes
-  package:
-    name:
-      - python3-pip
-      - python-pip
-    state: present
-
-- name: Install validations deps
-  ignore_errors: yes
-  shell:
-    cmd: |
-      sudo pip3 install prettytable
-      sudo pip3 install setuptools
-      sudo pip3 install ansible
-
-- name: Install validations deps
-  shell:
-    cmd: |
-      sudo pip install prettytable
-      sudo pip install setuptools
-      sudo pip3 install ansible
-
-- name: Install validations-libs
-  ignore_errors: yes
-  shell:
-    cmd: |
-      sudo pip install -r requirements.txt
-      sudo python setup.py install
-    executable: /bin/bash
-    chdir: '{{ src_dir }}/validations-libs'
-
-- name: Install validations-common
-  ignore_errors: yes
-  shell:
-    cmd: |
-      sudo pip install -r requirements.txt
-      sudo python setup.py install
-    executable: /bin/bash
-    chdir: '{{ src_dir }}/validations-common'
-
-- name: Make sure Validations Log dir exists
-  become: yes
-  file:
-    path: '{{ log_dir }}'
-    state: directory
-    mode: '0755'
-    owner: '{{ ansible_user }}'
-    group: '{{ ansible_user }}'
-    recurse: yes
-
-- name: Make validation script executable
-  register: output
-  shell:
-    cmd: |
-      chmod +x validations_common/validation.py
-    executable: /bin/bash
-    chdir: '{{ src_dir }}/validations-common'
-  environment: '{{ zuul | zuul_legacy_vars }}'

playbooks/post.yaml

@@ -0,0 +1,9 @@
+- hosts: all
+  vars:
+    tox_envlist: functional
+  roles:
+    - fetch-subunit-output
+    - fetch-devstack-log-dir
+    - fetch-output
+    - fetch-validations
+    - fetch-python-sdist-output

playbooks/validations-libs-functional.yaml

@@ -0,0 +1,14 @@
+---
+- hosts: tempest
+  name: validations-libs-functional
+  roles:
+    - ensure-tox
+    - ensure-pip
+    - ensure-virtualenv
+    - role: ensure-if-python
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/validations-libs"
+    - role: ensure-if-python
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/validations-common"
+    - role: validations

roles/fetch-validations/defaults/main.yaml

@@ -0,0 +1,3 @@
+src_dir: "{{ zuul.project.src_dir }}"
+output_dir: "{{ ansible_user_dir }}"
+log_path: "{{ zuul.executor.log_root }}"

roles/fetch-validations/tasks/main.yaml

@@ -0,0 +1,24 @@
+- name: Ensure local output dirs
+  delegate_to: localhost
+  file:
+    path: "{{ vf_output_dir }}"
+    state: directory
+  with_items:
+    - "{{ log_path }}/validations-logs"
+    - "{{ zuul.executor.work_root }}/artifacts"
+  loop_control:
+    loop_var: vf_output_dir
+
+- name: Collect logs and artifacts
+  synchronize:
+    dest: "{{ vf_output.dest }}/"
+    mode: pull
+    src: "{{ output_dir }}/{{ vf_output.src }}/"
+    verify_host: true
+    owner: false
+    group: false
+  loop:
+    - dest: "{{ log_path }}/validations-logs"
+      src: "logs"
+  loop_control:
+    loop_var: vf_output

roles/validations/defaults/main.yaml

@@ -0,0 +1,23 @@
+log_dir: "{{ ansible_user_dir }}/logs"
+zuul_work_virtualenv: "{{ ansible_user_dir }}/.venv"
+ansible_dir: "{{ zuul_work_virtualenv }}/share/validations-common"
+validation_dir: "{{ zuul_work_virtualenv }}/share/validations-common/playbooks"
+vf_log_dir: "/var/log/validations"
+val_exec: "source {{ zuul_work_virtualenv }}/bin/activate; validation.py"
+command:
+  - { output: "{{ log_dir }}/run.log", command: "{{ val_exec }} run --validation  check-ftype,512e \
+     --validation-dir {{ validation_dir }} \
+     --ansible-base-dir {{ ansible_dir }} \
+     --output-log {{ log_dir }}/run.log" }
+  - { output: "{{ log_dir }}/run-group.log", command: "{{ val_exec }} run --group prep \
+     --validation-dir {{ validation_dir }} \
+     --ansible-base-dir {{ ansible_dir }} \
+     --output-log {{ log_dir }}/run-group.log" }
+  - { output: "{{ log_dir }}/list.log", command: "{{ val_exec }} list \
+     --validation-dir {{ validation_dir }} \
+     --ansible-base-dir {{ ansible_dir }} \
+     --output-log {{ log_dir }}/list.log" }
+  - { output: "{{ log_dir }}/show.log", command: "{{ val_exec }} show \
+     --validation-dir {{ validation_dir }} \
+     --ansible-base-dir {{ ansible_dir }} \
+     --output-log {{ log_dir }}/show.log" }

roles/validations/tasks/main.yaml

@@ -0,0 +1,29 @@
+- name: Make sure Validations Log dir exists
+  become: yes
+  file:
+    path: "{{ vf_log_dir }}"
+    state: directory
+    mode: "0755"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    recurse: yes
+
+# @matbu: TODO make the /var/log/validations as a parameter
+# We should be able to write validation logs outside of the /var/log path
+# This statement will have to be remove then.
+- name: Create Validations log symlink
+  file:
+    src: "{{ vf_log_dir }}"
+    dest: "{{ log_dir }}"
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    state: link
+
+# @matbu: TODO make ansible-runner depends on Ansible
+- name: Install Ansible
+  pip:
+    name: ansible
+    virtualenv: "{{ zuul_work_virtualenv }}"
+
+- include: validations.yaml validation_command="{{ item.command }}" val_output="{{ item.output }}"
+  loop: "{{command}}"

roles/validations/tasks/validations.yaml

@@ -1,10 +1,12 @@
 - name: Execute validation commands
-  shell: "{{ validation_command }}"
+  shell:
+    cmd: "{{ validation_command }}"
+    executable: /bin/bash
 
 - name: Get run results
   register: result
   shell:
-    cmd: "cat {{val_output}}"
+    cmd: "cat {{ val_output }}"
     executable: /bin/bash
 
 - name: Get json data