Revert "download-artifact : support recursive download"
This reverts commit 7101fe7d1c13250415f5c6f6392c2a22720bbe43. This unfortunately has a number of problems. Firstly, the "Fail if no wget" fails when download_artifact_recurse isn't set, because we didn't check for wget. Also, the download doesn't work with some providers. wget asks for gzip downloads with it's accept headers (which can't be turned off) but the recursive download doesn't understand the gzipped index.html file and thus doesn't find anything to walk. The "--compression=auto" flag is available to overcome this, but is not widely supported (and not supported on the executor). https://review.opendev.org/733728 attempted to work-around this but the problems with this approach seem too much for now. Change-Id: I9bc55d771ec1828d374684d0ffe5ec1d1494773e
This commit is contained in:
Ian Wienand
parent
2c0eaed81d
commit
f946763d0a
@ -32,16 +32,3 @@ many artifacts as match the selection criteria.
|
|||||||
:default: {{ zuul.executor.work_root }}
|
:default: {{ zuul.executor.work_root }}
|
||||||
|
|
||||||
The directory in which to place the downloaded artifacts.
|
The directory in which to place the downloaded artifacts.
|
||||||
|
|
||||||
.. zuul:rolevar:: download_artifact_recurse
|
|
||||||
:default: False
|
|
||||||
|
|
||||||
If the ``type`` describes a directory, this can be used to download
|
|
||||||
it recursively. Note this requires ``wget`` on the host. The
|
|
||||||
*last* component of the artifact URL should be a directory, and
|
|
||||||
that will be the directory saved to
|
|
||||||
``download_artifact_directory``.
|
|
||||||
|
|
||||||
For example, the contents of
|
|
||||||
``https://foo.com/abc/123/test/the_artifact/`` will be saved to
|
|
||||||
``{{ download_artifact_directory }}/the_artifact/``.
|
|
||||||
|
|||||||
@ -1,4 +1,3 @@
|
|||||||
---
|
---
|
||||||
download_artifact_query: "change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}"
|
download_artifact_query: "change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}"
|
||||||
download_artifact_directory: "{{ zuul.executor.work_root }}"
|
download_artifact_directory: "{{ zuul.executor.work_root }}"
|
||||||
download_artifact_recurse: False
|
|
||||||
|
|||||||
@ -1,24 +1,3 @@
|
|||||||
- name: Look for wget
|
|
||||||
shell: |
|
|
||||||
command -v wget || exit 1
|
|
||||||
args:
|
|
||||||
executable: /bin/bash
|
|
||||||
register: _wget_installed
|
|
||||||
failed_when: false
|
|
||||||
when: download_artifact_recurse
|
|
||||||
|
|
||||||
- name: Fail if no wget
|
|
||||||
fail:
|
|
||||||
msg: 'wget not found, can not recurse'
|
|
||||||
when:
|
|
||||||
- _wget_installed.rc != 0
|
|
||||||
- download_artifact_recurse
|
|
||||||
|
|
||||||
- name: Ensure the download directory
|
|
||||||
file:
|
|
||||||
name: '{{ download_artifact_directory }}'
|
|
||||||
state: directory
|
|
||||||
|
|
||||||
- name: Query Zuul API for artifact information
|
- name: Query Zuul API for artifact information
|
||||||
uri:
|
uri:
|
||||||
url: "{{ download_artifact_api }}/builds?{{ download_artifact_query }}"
|
url: "{{ download_artifact_api }}/builds?{{ download_artifact_query }}"
|
||||||
@ -26,21 +5,6 @@
|
|||||||
- name: Parse build response
|
- name: Parse build response
|
||||||
set_fact:
|
set_fact:
|
||||||
build: "{{ build.json[0] }}"
|
build: "{{ build.json[0] }}"
|
||||||
|
|
||||||
- name: Recursive downloads
|
|
||||||
loop: "{{ build.artifacts }}"
|
|
||||||
include_tasks: recursive.yaml
|
|
||||||
loop_control:
|
|
||||||
loop_var: zj_artifact
|
|
||||||
when:
|
|
||||||
- "'metadata' in zj_artifact"
|
|
||||||
- "'type' in zj_artifact.metadata"
|
|
||||||
- download_artifact_recurse
|
|
||||||
- >
|
|
||||||
zj_artifact.metadata.type == download_artifact_type or
|
|
||||||
((download_artifact_type | type_debug) == 'list'
|
|
||||||
and zj_artifact.metadata.type in download_artifact_type)
|
|
||||||
|
|
||||||
- name: Download archive by type
|
- name: Download archive by type
|
||||||
uri:
|
uri:
|
||||||
url: "{{ zj_artifact.url }}"
|
url: "{{ zj_artifact.url }}"
|
||||||
|
|||||||
@ -1,27 +0,0 @@
|
|||||||
- name: Find directories to strip
|
|
||||||
set_fact:
|
|
||||||
# - take the url, strip of any trailing "/"
|
|
||||||
# - find everything on the // side
|
|
||||||
# - split it at /'s, then disregrad the host (ignored by parent
|
|
||||||
# flag) and the final directory (the one we want) i.e. -2
|
|
||||||
cut_dirs: '{{ zj_artifact.url.rpartition("//")[-1].rstrip("/").split("/") | length - 2 }}'
|
|
||||||
|
|
||||||
- name: Recursive download
|
|
||||||
# NOTE(ianw): something weird about wget recursive, you want to make
|
|
||||||
# sure the directory to download has a trailing "/" or you can get
|
|
||||||
# directory errors; something like
|
|
||||||
# http://savannah.gnu.org/bugs/?29647
|
|
||||||
# Otherwise this
|
|
||||||
# - drops the host (-nH)
|
|
||||||
# - doesn't walk upwards (--no-parent)
|
|
||||||
# - doesn't save index.html files (maybe this should be optional for docs?)
|
|
||||||
# - cuts out the all the parent directories but the last one
|
|
||||||
#
|
|
||||||
# so "https://foo.com/logs/123/abc/dir/the_artifact" gets saved to
|
|
||||||
# {{ download_artifact_directory }}/the_artifact
|
|
||||||
command: |
|
|
||||||
wget -nH --no-parent --reject="index.html*" --cut-dirs={{ cut_dirs }} -P {{ download_artifact_directory }} -m {{ zj_artifact.url.rstrip("/") + "/" }}
|
|
||||||
args:
|
|
||||||
# We really want to use wget here, because the get_url type things
|
|
||||||
# don't do recursive.
|
|
||||||
warn: false
|
|
||||||
@ -1,35 +0,0 @@
|
|||||||
- name: Run the download-artifacts role
|
|
||||||
hosts: all
|
|
||||||
roles:
|
|
||||||
- ensure-output-dirs
|
|
||||||
|
|
||||||
tasks:
|
|
||||||
# NOTE(ianw) : This has probably expired and thus broken this
|
|
||||||
# test. You need to choose a new artifact to download to test
|
|
||||||
# this as part of updating the download-artifact role
|
|
||||||
|
|
||||||
- name: Download centos 7 x86 artifact on host
|
|
||||||
include_role:
|
|
||||||
name: download-artifact
|
|
||||||
vars:
|
|
||||||
download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
|
|
||||||
download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
|
|
||||||
download_artifact_type: rpm
|
|
||||||
download_artifact_pipeline: gate
|
|
||||||
download_artifact_job: openafs-rpm-package-build-centos-7-x86
|
|
||||||
download_artifact_directory: '/tmp/'
|
|
||||||
download_artifact_recurse: yes
|
|
||||||
|
|
||||||
- name: Download centos 7 x86 artifact on executor
|
|
||||||
include_role:
|
|
||||||
name: download-artifact
|
|
||||||
apply:
|
|
||||||
delegate_to: localhost
|
|
||||||
vars:
|
|
||||||
download_artifact_query: "change=733420&patchset=1&pipeline=gate&job_name=openafs-rpm-package-build-centos-7-x86"
|
|
||||||
download_artifact_api: "https://zuul.opendev.org/api/tenant/openstack"
|
|
||||||
download_artifact_type: rpm
|
|
||||||
download_artifact_pipeline: gate
|
|
||||||
download_artifact_job: openafs-rpm-package-build-centos-7-x86
|
|
||||||
download_artifact_directory: '{{ zuul.executor.work_root }}/test-artifacts/'
|
|
||||||
download_artifact_recurse: yes
|
|
||||||
@ -1,159 +0,0 @@
|
|||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact
|
|
||||||
tags: all-platforms
|
|
||||||
description: |
|
|
||||||
Test artifact roles
|
|
||||||
parent: base-minimal
|
|
||||||
run: test-playbooks/download-artifact.yaml
|
|
||||||
files:
|
|
||||||
- ^roles/download-artifact/.*
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-centos-7
|
|
||||||
description: Test artifact roles on centos-7
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: centos-7
|
|
||||||
label: centos-7
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-centos-8
|
|
||||||
description: Test artifact roles on centos-8
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: centos-8
|
|
||||||
label: centos-8
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-debian-stretch
|
|
||||||
description: Test artifact roles on debian-stretch
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: debian-stretch
|
|
||||||
label: debian-stretch
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-fedora-31
|
|
||||||
description: Test artifact roles on fedora-31
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: fedora-31
|
|
||||||
label: fedora-31
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
|
||||||
description: Test artifact roles on gentoo-17-0-systemd
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: gentoo-17-0-systemd
|
|
||||||
label: gentoo-17-0-systemd
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-opensuse-15
|
|
||||||
description: Test artifact roles on opensuse-15
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: opensuse-15
|
|
||||||
label: opensuse-15
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
|
|
||||||
voting: false
|
|
||||||
description: Test artifact roles on opensuse-tumbleweed
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: opensuse-tumbleweed
|
|
||||||
label: opensuse-tumbleweed
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-ubuntu-bionic
|
|
||||||
description: Test artifact roles on ubuntu-bionic
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: ubuntu-bionic
|
|
||||||
label: ubuntu-bionic
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-ubuntu-xenial
|
|
||||||
description: Test artifact roles on ubuntu-xenial
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: ubuntu-xenial
|
|
||||||
label: ubuntu-xenial
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
|
||||||
description: Test artifact roles on ubuntu-bionic-plain
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: ubuntu-bionic-plain
|
|
||||||
label: ubuntu-bionic-plain
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
|
||||||
description: Test artifact roles on ubuntu-xenial-plain
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: ubuntu-xenial-plain
|
|
||||||
label: ubuntu-xenial-plain
|
|
||||||
|
|
||||||
- job:
|
|
||||||
name: zuul-jobs-test-download-artifact-centos-8-plain
|
|
||||||
description: Test artifact roles on centos-8-plain
|
|
||||||
parent: zuul-jobs-test-download-artifact
|
|
||||||
tags: auto-generated
|
|
||||||
nodeset:
|
|
||||||
nodes:
|
|
||||||
- name: centos-8-plain
|
|
||||||
label: centos-8-plain
|
|
||||||
|
|
||||||
- project:
|
|
||||||
check:
|
|
||||||
jobs:
|
|
||||||
- zuul-jobs-test-download-artifact-centos-7
|
|
||||||
- zuul-jobs-test-download-artifact-centos-8
|
|
||||||
- zuul-jobs-test-download-artifact-debian-stretch
|
|
||||||
- zuul-jobs-test-download-artifact-fedora-31
|
|
||||||
- zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
|
||||||
- zuul-jobs-test-download-artifact-opensuse-15
|
|
||||||
- zuul-jobs-test-download-artifact-opensuse-tumbleweed-nv
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-bionic
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-xenial
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
|
||||||
- zuul-jobs-test-download-artifact-centos-8-plain
|
|
||||||
gate:
|
|
||||||
jobs:
|
|
||||||
- zuul-jobs-test-download-artifact-centos-7
|
|
||||||
- zuul-jobs-test-download-artifact-centos-8
|
|
||||||
- zuul-jobs-test-download-artifact-debian-stretch
|
|
||||||
- zuul-jobs-test-download-artifact-fedora-31
|
|
||||||
- zuul-jobs-test-download-artifact-gentoo-17-0-systemd
|
|
||||||
- zuul-jobs-test-download-artifact-opensuse-15
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-bionic
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-xenial
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-bionic-plain
|
|
||||||
- zuul-jobs-test-download-artifact-ubuntu-xenial-plain
|
|
||||||
- zuul-jobs-test-download-artifact-centos-8-plain
|
|
||||||
Loading…
x
Reference in New Issue
Block a user