diff --git a/roles/prepare-workspace-git/README.rst b/roles/prepare-workspace-git/README.rst index 1cc5ce270..4faf51919 100644 --- a/roles/prepare-workspace-git/README.rst +++ b/roles/prepare-workspace-git/README.rst @@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: prepare_workspace_sync_required_projects_only + :type: bool + :default: False + + A flag which if set to true, filters the to be synchronized project + list to only use projects which are required by the job. + .. zuul:rolevar:: mirror_workspace_quiet :default: false diff --git a/roles/prepare-workspace-git/defaults/main.yaml b/roles/prepare-workspace-git/defaults/main.yaml index f6e5da9bb..fbd1b27ef 100644 --- a/roles/prepare-workspace-git/defaults/main.yaml +++ b/roles/prepare-workspace-git/defaults/main.yaml @@ -1,3 +1,4 @@ cached_repos_root: /opt/git mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" +prepare_workspace_sync_required_projects_only: false diff --git a/roles/prepare-workspace-git/tasks/main.yaml b/roles/prepare-workspace-git/tasks/main.yaml index d57ef11f1..5e68daf99 100644 --- a/roles/prepare-workspace-git/tasks/main.yaml +++ b/roles/prepare-workspace-git/tasks/main.yaml @@ -1,3 +1,20 @@ +- name: Filter zuul projects if sync-only-required-projects flag is set + set_fact: + _zuul_projects: > + {{ _zuul_projects | default({}) | + combine({ zj_project.key : zj_project.value }) }} + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + when: + - prepare_workspace_sync_required_projects_only + - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required + +- name: Don't filter zuul projects if flag is false + set_fact: + _zuul_projects: "{{ zuul.projects }}" + when: not prepare_workspace_sync_required_projects_only + # Do all the steps in a single shell script. This reduces the number of times # ansible must loop over the list of projects which reduces the amount of # task startup time we incur. @@ -17,7 +34,7 @@ git remote add origin file:///dev/null args: creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" - with_items: "{{ zuul.projects.values() | list }}" + with_items: "{{ _zuul_projects.values() }}" loop_control: loop_var: zj_project # We're using git in a shell script because it is faster and the module @@ -31,7 +48,7 @@ value: ignore scope: local repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project @@ -46,7 +63,7 @@ chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" environment: GIT_ALLOW_PROTOCOL: ext:ssh - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project delegate_to: localhost @@ -80,7 +97,7 @@ git log --pretty=oneline -1 args: chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, diff --git a/roles/test-prepare-workspace-git/README.rst b/roles/test-prepare-workspace-git/README.rst index 1cc5ce270..4faf51919 100644 --- a/roles/test-prepare-workspace-git/README.rst +++ b/roles/test-prepare-workspace-git/README.rst @@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: prepare_workspace_sync_required_projects_only + :type: bool + :default: False + + A flag which if set to true, filters the to be synchronized project + list to only use projects which are required by the job. + .. zuul:rolevar:: mirror_workspace_quiet :default: false diff --git a/roles/test-prepare-workspace-git/defaults/main.yaml b/roles/test-prepare-workspace-git/defaults/main.yaml index f6e5da9bb..fbd1b27ef 100644 --- a/roles/test-prepare-workspace-git/defaults/main.yaml +++ b/roles/test-prepare-workspace-git/defaults/main.yaml @@ -1,3 +1,4 @@ cached_repos_root: /opt/git mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" +prepare_workspace_sync_required_projects_only: false diff --git a/roles/test-prepare-workspace-git/tasks/main.yaml b/roles/test-prepare-workspace-git/tasks/main.yaml index d57ef11f1..5e68daf99 100644 --- a/roles/test-prepare-workspace-git/tasks/main.yaml +++ b/roles/test-prepare-workspace-git/tasks/main.yaml @@ -1,3 +1,20 @@ +- name: Filter zuul projects if sync-only-required-projects flag is set + set_fact: + _zuul_projects: > + {{ _zuul_projects | default({}) | + combine({ zj_project.key : zj_project.value }) }} + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + when: + - prepare_workspace_sync_required_projects_only + - zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required + +- name: Don't filter zuul projects if flag is false + set_fact: + _zuul_projects: "{{ zuul.projects }}" + when: not prepare_workspace_sync_required_projects_only + # Do all the steps in a single shell script. This reduces the number of times # ansible must loop over the list of projects which reduces the amount of # task startup time we incur. @@ -17,7 +34,7 @@ git remote add origin file:///dev/null args: creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" - with_items: "{{ zuul.projects.values() | list }}" + with_items: "{{ _zuul_projects.values() }}" loop_control: loop_var: zj_project # We're using git in a shell script because it is faster and the module @@ -31,7 +48,7 @@ value: ignore scope: local repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project @@ -46,7 +63,7 @@ chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" environment: GIT_ALLOW_PROTOCOL: ext:ssh - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project delegate_to: localhost @@ -80,7 +97,7 @@ git log --pretty=oneline -1 args: chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" - with_dict: "{{ zuul.projects }}" + with_dict: "{{ _zuul_projects }}" loop_control: loop_var: zj_project # ANSIBLE0006: Skip linting since it triggers on the "git" command, diff --git a/test-playbooks/base-roles/base.yaml b/test-playbooks/base-roles/base.yaml index d760f4af6..1c5e93521 100644 --- a/test-playbooks/base-roles/base.yaml +++ b/test-playbooks/base-roles/base.yaml @@ -6,6 +6,7 @@ # Note: set-zuul-log-path-fact is tested by emit-job-header.yaml - import_playbook: emit-job-header.yaml - import_playbook: ensure-output-dirs.yaml +- import_playbook: prepare-workspace-git-required-projects-only.yaml - import_playbook: prepare-workspace-git.yaml - import_playbook: configure-mirrors.yaml - import_playbook: fetch-zuul-cloner.yaml diff --git a/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml new file mode 100644 index 000000000..196364c9d --- /dev/null +++ b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml @@ -0,0 +1,6 @@ +- name: Test the prepare-workspace-git role + hosts: all + roles: + - role: prepare-workspace-git + vars: + prepare_workspace_sync_required_projects_only: true diff --git a/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml new file mode 100644 index 000000000..8967ba2f7 --- /dev/null +++ b/test-playbooks/base-roles/prepare-workspace-git-required-projects-only.yaml @@ -0,0 +1,72 @@ +- name: Prepare to test the prepare-workspace-git role with sync required only + hosts: all + tasks: + - name: Delete remote source directory to start with clean state + file: + state: absent + path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + +# We need to override the zuul.projects variable, and that is not +# possible in a Zuul job. So we use a nested Ansible to perform this +# test. +- name: Test the prepare-workspace-git role with sync required only + hosts: localhost + vars: + # Mutate the zuul vars supplied to this test job to simulate a + # repo being included as non-required (i.e., a depends-on). + zuul_mod: + projects: + opendev.org/zuul/project-config: + required: false + tasks: + - name: Create nested zuul vars + set_fact: + nested_zuul: + zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}" + - name: Write nested zuul vars + copy: + content: '{{ nested_zuul | to_nice_yaml(indent=2) }}' + dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml" + - name: Run nested Ansible + command: >- + {{ ansible_playbook_python | dirname}}/ansible-playbook + -vvv + -e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml + -e zuul_execution_phase=nested + -e zuul_execution_phase_index=0 + -e zuul_execution_canonical_name_and_path=opendev.org/zuul/zuul-jobs/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml + -e zuul_execution_trusted=False + -e zuul_execution_branch={{zuul_execution_branch}} + {{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml + environment: + ANSIBLE_ROLES_PATH: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles" + +- name: Verify the prepare-workspace-git role with sync required only + hosts: all + tasks: + # opendev/base-jobs is in 'required-projects'. + # Also check that the project being tested is being prepared. + # We're checking them explicitly rather than with_items on zuul.projects + # in case there is a regression which would take an item out. + - name: Check that opendev/base-jobs was prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs" + register: base_jobs + + - name: Check that zuul/project-config was not prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config" + register: project_config + + - name: Check this project was prepared + stat: + path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" + register: self_config + + - name: Validate that required projects have been prepared + assert: + that: + - base_jobs.stat.exists + - not project_config.stat.exists + - self_config.stat.exists diff --git a/test-playbooks/base-roles/prepare-workspace-git.yaml b/test-playbooks/base-roles/prepare-workspace-git.yaml index 6a83cfbdc..24a8a16d6 100644 --- a/test-playbooks/base-roles/prepare-workspace-git.yaml +++ b/test-playbooks/base-roles/prepare-workspace-git.yaml @@ -1,5 +1,12 @@ - name: Test the prepare-workspace-git role hosts: all + pre_tasks: + - name: Delete remote source directory to start with clean state + file: + state: absent + path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + roles: - role: prepare-workspace-git post_tasks: @@ -12,6 +19,11 @@ path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs" register: base_jobs + - name: Check that zuul/project-config was prepared + stat: + path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config" + register: project_config + - name: Check this project was prepared stat: path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" @@ -21,4 +33,5 @@ assert: that: - base_jobs.stat.exists + - project_config.stat.exists - self_config.stat.exists diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index 2e5cb9f5e..2fa0bbeef 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -116,10 +116,11 @@ tags: all-platforms abstract: true run: test-playbooks/base-roles/base.yaml - # Testing of fetch-zuul-cloner and use-cached-repos need this repo - # in required-projects + # Testing of fetch-zuul-cloner and prepare-workspace-git need + # these repos in required-projects required-projects: - opendev/base-jobs + - zuul/project-config files: - ^roles/configure-mirrors/.* - ^roles/emit-job-header/.*