Merge "download-artifact : support recursive download"

roles/download-artifact/README.rst

@@ -32,3 +32,16 @@ many artifacts as match the selection criteria.
    :default: {{ zuul.executor.work_root }}
 
    The directory in which to place the downloaded artifacts.
+
+.. zuul:rolevar:: download_artifact_recurse
+   :default: False
+
+   If the ``type`` describes a directory, this can be used to download
+   it recursively.  Note this requires ``wget`` on the host.  The
+   *last* component of the artifact URL should be a directory, and
+   that will be the directory saved to
+   ``download_artifact_directory``.
+
+   For example, the contents of
+   ``https://foo.com/abc/123/test/the_artifact/`` will be saved to
+   ``{{ download_artifact_directory }}/the_artifact/``.

roles/download-artifact/defaults/main.yaml

@@ -1,3 +1,4 @@
 ---
 download_artifact_query: "change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}"
 download_artifact_directory: "{{ zuul.executor.work_root }}"
+download_artifact_recurse: False

roles/download-artifact/tasks/main.yaml

@@ -1,3 +1,24 @@
+- name: Look for wget
+  shell: |
+      command -v wget || exit 1
+  args:
+    executable: /bin/bash
+  register: _wget_installed
+  failed_when: false
+  when: download_artifact_recurse
+
+- name: Fail if no wget
+  fail:
+    msg: 'wget not found, can not recurse'
+  when:
+    - _wget_installed.rc != 0
+    - download_artifact_recurse
+
+- name: Ensure the download directory
+  file:
+    name: '{{ download_artifact_directory }}'
+    state: directory
+
 - name: Query Zuul API for artifact information
   uri:
     url: "{{ download_artifact_api }}/builds?{{ download_artifact_query }}"
@@ -5,6 +26,14 @@
 - name: Parse build response
   set_fact:
     build: "{{ build.json[0] }}"
+
+- name: Recursive downloads
+  loop: "{{ build.artifacts }}"
+  include_tasks: recursive.yaml
+  loop_control:
+    loop_var: zj_artifact
+  when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type)) and download_artifact_recurse"
+
 - name: Download archive by type
   uri:
     url: "{{ zj_artifact.url }}"
@@ -12,4 +41,4 @@
   loop: "{{ build.artifacts }}"
   loop_control:
     loop_var: zj_artifact
-  when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type))"
+  when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type)) and not download_artifact_recurse"

roles/download-artifact/tasks/recursive.yaml

@@ -0,0 +1,27 @@
+- name: Find directories to strip
+  set_fact:
+    # - take the url, strip of any trailing "/"
+    # - find everything on the // side
+    # - split it at /'s, then disregrad the host (ignored by parent
+    #   flag) and the final directory (the one we want) i.e. -2
+    cut_dirs: '{{ zj_artifact.url.rpartition("//")[-1].rstrip("/").split("/") | length - 2 }}'
+
+- name: Recursive download
+  # NOTE(ianw): something weird about wget recursive, you want to make
+  # sure the directory to download has a trailing "/" or you can get
+  # directory errors; something like
+  #  http://savannah.gnu.org/bugs/?29647
+  # Otherwise this
+  #  - drops the host (-nH)
+  #  - doesn't walk upwards (--no-parent)
+  #  - doesn't save index.html files (maybe this should be optional for docs?)
+  #  - cuts out the all the parent directories but the last one
+  #
+  #  so "https://foo.com/logs/123/abc/dir/the_artifact" gets saved to
+  #  {{ download_artifact_directory }}/the_artifact
+  command: |
+    wget -nH --no-parent --reject="index.html*" --cut-dirs={{ cut_dirs }} -P {{ download_artifact_directory }} -m {{ zj_artifact.url.rstrip("/") + "/" }}
+  args:
+    # We really want to use wget here, because the get_url type things
+    # don't do recursive.
+    warn: false

test-playbooks/download-artifact.yaml

@@ -0,0 +1,35 @@
+- name: Run the download-artifacts role
+  hosts: all
+  roles:
+    - ensure-output-dirs
+
+  tasks:
+    # NOTE(ianw) : This has probably expired and thus broken this
+    # test.  You need to choose a new artifact to download to test
+    # this as part of updating the download-artifact role
+
+    - name: Download centos 7 x86 artifact on host
+      include_role:
+        name: download-artifact
+      vars:
+        download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
+        download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
+        download_artifact_type: rpm
+        download_artifact_pipeline: gate
+        download_artifact_job: openafs-rpm-package-build-centos-7-x86
+        download_artifact_directory: '/tmp/'
+        download_artifact_recurse: yes
+
+    - name: Download centos 7 x86 artifact on executor
+      include_role:
+        name: download-artifact
+        apply:
+          delegate_to: localhost
+      vars:
+        download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
+        download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
+        download_artifact_type: rpm
+        download_artifact_pipeline: gate
+        download_artifact_job: openafs-rpm-package-build-centos-7-x86
+        download_artifact_directory: '{{ zuul.executor.work_root }}/test-artifacts/'
+        download_artifact_recurse: yes

zuul-tests.d/artifact-jobs.yaml

@@ -0,0 +1,159 @@
+- job:
+    name: zuul-jobs-test-download-artifact
+    tags: all-platforms
+    description: |
+      Test artifact roles
+    parent: base-minimal
+    run: test-playbooks/download-artifact.yaml
+    files:
+      - ^roles/download-artifact/.*
+
+- job:
+    name: zuul-jobs-test-download-artifact-centos-7
+    description: Test artifact roles on centos-7
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-7
+          label: centos-7
+
+- job:
+    name: zuul-jobs-test-download-artifact-centos-8
+    description: Test artifact roles on centos-8
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8
+          label: centos-8
+
+- job:
+    name: zuul-jobs-test-download-artifact-debian-stretch
+    description: Test artifact roles on debian-stretch
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-stretch
+          label: debian-stretch
+
+- job:
+    name: zuul-jobs-test-download-artifact-fedora-31
+    description: Test artifact roles on fedora-31
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: fedora-31
+          label: fedora-31
+
+- job:
+    name: zuul-jobs-test-download-artifact-gentoo-17-0-systemd
+    description: Test artifact roles on gentoo-17-0-systemd
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: gentoo-17-0-systemd
+          label: gentoo-17-0-systemd
+
+- job:
+    name: zuul-jobs-test-download-artifact-opensuse-15
+    description: Test artifact roles on opensuse-15
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-15
+          label: opensuse-15
+
+- job:
+    name: zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
+    voting: false
+    description: Test artifact roles on opensuse-tumbleweed
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-tumbleweed
+          label: opensuse-tumbleweed
+
+- job:
+    name: zuul-jobs-test-download-artifact-ubuntu-bionic
+    description: Test artifact roles on ubuntu-bionic
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-download-artifact-ubuntu-xenial
+    description: Test artifact roles on ubuntu-xenial
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial
+          label: ubuntu-xenial
+
+- job:
+    name: zuul-jobs-test-download-artifact-ubuntu-bionic-plain
+    description: Test artifact roles on ubuntu-bionic-plain
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic-plain
+          label: ubuntu-bionic-plain
+
+- job:
+    name: zuul-jobs-test-download-artifact-ubuntu-xenial-plain
+    description: Test artifact roles on ubuntu-xenial-plain
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial-plain
+          label: ubuntu-xenial-plain
+
+- job:
+    name: zuul-jobs-test-download-artifact-centos-8-plain
+    description: Test artifact roles on centos-8-plain
+    parent: zuul-jobs-test-download-artifact
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8-plain
+          label: centos-8-plain
+
+- project:
+    check:
+      jobs:
+        - zuul-jobs-test-download-artifact-centos-7
+        - zuul-jobs-test-download-artifact-centos-8
+        - zuul-jobs-test-download-artifact-debian-stretch
+        - zuul-jobs-test-download-artifact-fedora-31
+        - zuul-jobs-test-download-artifact-gentoo-17-0-systemd
+        - zuul-jobs-test-download-artifact-opensuse-15
+        - zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
+        - zuul-jobs-test-download-artifact-ubuntu-bionic
+        - zuul-jobs-test-download-artifact-ubuntu-xenial
+        - zuul-jobs-test-download-artifact-ubuntu-bionic-plain
+        - zuul-jobs-test-download-artifact-ubuntu-xenial-plain
+        - zuul-jobs-test-download-artifact-centos-8-plain
+    gate:
+      jobs:
+        - zuul-jobs-test-download-artifact-centos-7
+        - zuul-jobs-test-download-artifact-centos-8
+        - zuul-jobs-test-download-artifact-debian-stretch
+        - zuul-jobs-test-download-artifact-fedora-31
+        - zuul-jobs-test-download-artifact-gentoo-17-0-systemd
+        - zuul-jobs-test-download-artifact-opensuse-15
+        - zuul-jobs-test-download-artifact-ubuntu-bionic
+        - zuul-jobs-test-download-artifact-ubuntu-xenial
+        - zuul-jobs-test-download-artifact-ubuntu-bionic-plain
+        - zuul-jobs-test-download-artifact-ubuntu-xenial-plain
+        - zuul-jobs-test-download-artifact-centos-8-plain