Merge "download-artifact : support recursive download"
This commit is contained in:
Zuul
Gerrit Code Review
commit
eed60ad916
@ -32,3 +32,16 @@ many artifacts as match the selection criteria.
|
||||
:default: {{ zuul.executor.work_root }}
|
||||
|
||||
The directory in which to place the downloaded artifacts.
|
||||
|
||||
.. zuul:rolevar:: download_artifact_recurse
|
||||
:default: False
|
||||
|
||||
If the ``type`` describes a directory, this can be used to download
|
||||
it recursively. Note this requires ``wget`` on the host. The
|
||||
*last* component of the artifact URL should be a directory, and
|
||||
that will be the directory saved to
|
||||
``download_artifact_directory``.
|
||||
|
||||
For example, the contents of
|
||||
``https://foo.com/abc/123/test/the_artifact/`` will be saved to
|
||||
``{{ download_artifact_directory }}/the_artifact/``.
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
---
|
||||
download_artifact_query: "change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}"
|
||||
download_artifact_directory: "{{ zuul.executor.work_root }}"
|
||||
download_artifact_recurse: False
|
||||
|
||||
@ -1,3 +1,24 @@
|
||||
- name: Look for wget
|
||||
shell: |
|
||||
command -v wget || exit 1
|
||||
args:
|
||||
executable: /bin/bash
|
||||
register: _wget_installed
|
||||
failed_when: false
|
||||
when: download_artifact_recurse
|
||||
|
||||
- name: Fail if no wget
|
||||
fail:
|
||||
msg: 'wget not found, can not recurse'
|
||||
when:
|
||||
- _wget_installed.rc != 0
|
||||
- download_artifact_recurse
|
||||
|
||||
- name: Ensure the download directory
|
||||
file:
|
||||
name: '{{ download_artifact_directory }}'
|
||||
state: directory
|
||||
|
||||
- name: Query Zuul API for artifact information
|
||||
uri:
|
||||
url: "{{ download_artifact_api }}/builds?{{ download_artifact_query }}"
|
||||
@ -5,6 +26,14 @@
|
||||
- name: Parse build response
|
||||
set_fact:
|
||||
build: "{{ build.json[0] }}"
|
||||
|
||||
- name: Recursive downloads
|
||||
loop: "{{ build.artifacts }}"
|
||||
include_tasks: recursive.yaml
|
||||
loop_control:
|
||||
loop_var: zj_artifact
|
||||
when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type)) and download_artifact_recurse"
|
||||
|
||||
- name: Download archive by type
|
||||
uri:
|
||||
url: "{{ zj_artifact.url }}"
|
||||
@ -12,4 +41,4 @@
|
||||
loop: "{{ build.artifacts }}"
|
||||
loop_control:
|
||||
loop_var: zj_artifact
|
||||
when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type))"
|
||||
when: "'metadata' in zj_artifact and 'type' in zj_artifact.metadata and (zj_artifact.metadata.type == download_artifact_type or ((download_artifact_type | type_debug) == 'list' and zj_artifact.metadata.type in download_artifact_type)) and not download_artifact_recurse"
|
||||
|
||||
27
roles/download-artifact/tasks/recursive.yaml
Normal file
27
roles/download-artifact/tasks/recursive.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
- name: Find directories to strip
|
||||
set_fact:
|
||||
# - take the url, strip of any trailing "/"
|
||||
# - find everything on the // side
|
||||
# - split it at /'s, then disregrad the host (ignored by parent
|
||||
# flag) and the final directory (the one we want) i.e. -2
|
||||
cut_dirs: '{{ zj_artifact.url.rpartition("//")[-1].rstrip("/").split("/") | length - 2 }}'
|
||||
|
||||
- name: Recursive download
|
||||
# NOTE(ianw): something weird about wget recursive, you want to make
|
||||
# sure the directory to download has a trailing "/" or you can get
|
||||
# directory errors; something like
|
||||
# http://savannah.gnu.org/bugs/?29647
|
||||
# Otherwise this
|
||||
# - drops the host (-nH)
|
||||
# - doesn't walk upwards (--no-parent)
|
||||
# - doesn't save index.html files (maybe this should be optional for docs?)
|
||||
# - cuts out the all the parent directories but the last one
|
||||
#
|
||||
# so "https://foo.com/logs/123/abc/dir/the_artifact" gets saved to
|
||||
# {{ download_artifact_directory }}/the_artifact
|
||||
command: |
|
||||
wget -nH --no-parent --reject="index.html*" --cut-dirs={{ cut_dirs }} -P {{ download_artifact_directory }} -m {{ zj_artifact.url.rstrip("/") + "/" }}
|
||||
args:
|
||||
# We really want to use wget here, because the get_url type things
|
||||
# don't do recursive.
|
||||
warn: false
|
||||
35
test-playbooks/download-artifact.yaml
Normal file
35
test-playbooks/download-artifact.yaml
Normal file
@ -0,0 +1,35 @@
|
||||
- name: Run the download-artifacts role
|
||||
hosts: all
|
||||
roles:
|
||||
- ensure-output-dirs
|
||||
|
||||
tasks:
|
||||
# NOTE(ianw) : This has probably expired and thus broken this
|
||||
# test. You need to choose a new artifact to download to test
|
||||
# this as part of updating the download-artifact role
|
||||
|
||||
- name: Download centos 7 x86 artifact on host
|
||||
include_role:
|
||||
name: download-artifact
|
||||
vars:
|
||||
download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
|
||||
download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
|
||||
download_artifact_type: rpm
|
||||
download_artifact_pipeline: gate
|
||||
download_artifact_job: openafs-rpm-package-build-centos-7-x86
|
||||
download_artifact_directory: '/tmp/'
|
||||
download_artifact_recurse: yes
|
||||
|
||||
- name: Download centos 7 x86 artifact on executor
|
||||
include_role:
|
||||
name: download-artifact
|
||||
apply:
|
||||
delegate_to: localhost
|
||||
vars:
|
||||
download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
|
||||
download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
|
||||
download_artifact_type: rpm
|
||||
download_artifact_pipeline: gate
|
||||
download_artifact_job: openafs-rpm-package-build-centos-7-x86
|
||||
download_artifact_directory: '{{ zuul.executor.work_root }}/test-artifacts/'
|
||||
download_artifact_recurse: yes
|
||||
159
zuul-tests.d/artifact-jobs.yaml
Normal file
159
zuul-tests.d/artifact-jobs.yaml
Normal file
@ -0,0 +1,159 @@
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact
|
||||
tags: all-platforms
|
||||
description: |
|
||||
Test artifact roles
|
||||
parent: base-minimal
|
||||
run: test-playbooks/download-artifact.yaml
|
||||
files:
|
||||
- ^roles/download-artifact/.*
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-centos-7
|
||||
description: Test artifact roles on centos-7
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: centos-7
|
||||
label: centos-7
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-centos-8
|
||||
description: Test artifact roles on centos-8
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: centos-8
|
||||
label: centos-8
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-debian-stretch
|
||||
description: Test artifact roles on debian-stretch
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: debian-stretch
|
||||
label: debian-stretch
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-fedora-31
|
||||
description: Test artifact roles on fedora-31
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: fedora-31
|
||||
label: fedora-31
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
||||
description: Test artifact roles on gentoo-17-0-systemd
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: gentoo-17-0-systemd
|
||||
label: gentoo-17-0-systemd
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-opensuse-15
|
||||
description: Test artifact roles on opensuse-15
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: opensuse-15
|
||||
label: opensuse-15
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
|
||||
voting: false
|
||||
description: Test artifact roles on opensuse-tumbleweed
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: opensuse-tumbleweed
|
||||
label: opensuse-tumbleweed
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-ubuntu-bionic
|
||||
description: Test artifact roles on ubuntu-bionic
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: ubuntu-bionic
|
||||
label: ubuntu-bionic
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-ubuntu-xenial
|
||||
description: Test artifact roles on ubuntu-xenial
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: ubuntu-xenial
|
||||
label: ubuntu-xenial
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
||||
description: Test artifact roles on ubuntu-bionic-plain
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: ubuntu-bionic-plain
|
||||
label: ubuntu-bionic-plain
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
||||
description: Test artifact roles on ubuntu-xenial-plain
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: ubuntu-xenial-plain
|
||||
label: ubuntu-xenial-plain
|
||||
|
||||
- job:
|
||||
name: zuul-jobs-test-download-artifact-centos-8-plain
|
||||
description: Test artifact roles on centos-8-plain
|
||||
parent: zuul-jobs-test-download-artifact
|
||||
tags: auto-generated
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: centos-8-plain
|
||||
label: centos-8-plain
|
||||
|
||||
- project:
|
||||
check:
|
||||
jobs:
|
||||
- zuul-jobs-test-download-artifact-centos-7
|
||||
- zuul-jobs-test-download-artifact-centos-8
|
||||
- zuul-jobs-test-download-artifact-debian-stretch
|
||||
- zuul-jobs-test-download-artifact-fedora-31
|
||||
- zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
||||
- zuul-jobs-test-download-artifact-opensuse-15
|
||||
- zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
|
||||
- zuul-jobs-test-download-artifact-ubuntu-bionic
|
||||
- zuul-jobs-test-download-artifact-ubuntu-xenial
|
||||
- zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
||||
- zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
||||
- zuul-jobs-test-download-artifact-centos-8-plain
|
||||
gate:
|
||||
jobs:
|
||||
- zuul-jobs-test-download-artifact-centos-7
|
||||
- zuul-jobs-test-download-artifact-centos-8
|
||||
- zuul-jobs-test-download-artifact-debian-stretch
|
||||
- zuul-jobs-test-download-artifact-fedora-31
|
||||
- zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
||||
- zuul-jobs-test-download-artifact-opensuse-15
|
||||
- zuul-jobs-test-download-artifact-ubuntu-bionic
|
||||
- zuul-jobs-test-download-artifact-ubuntu-xenial
|
||||
- zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
||||
- zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
||||
- zuul-jobs-test-download-artifact-centos-8-plain
|
||||
Loading…
x
Reference in New Issue
Block a user