From ddc0da55a16ccfd9472dd8966373e1e9799b134d Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 22 Oct 2019 21:18:18 +1100 Subject: [PATCH] upload-logs-swift: test bulk download script change This implements the production change I98c80f657f38c5e1ed5f28e5d36988a3429ad1f8 in the test role. Review comments should be left there; we can merge this and then parent a job to base-test to test it. Change-Id: Id91350ff1c531fd7266f3bf76681a8415941481f --- roles/test-upload-logs-swift/README.rst | 14 +++ .../test-upload-logs-swift/defaults/main.yaml | 2 + .../test-fixtures/download-logs-sample.sh | 84 +++++++++++++++++ .../library/zuul_swift_upload.py | 89 +++++++++++++++---- roles/test-upload-logs-swift/tasks/main.yaml | 7 ++ 5 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 roles/test-upload-logs-swift/library/test-fixtures/download-logs-sample.sh diff --git a/roles/test-upload-logs-swift/README.rst b/roles/test-upload-logs-swift/README.rst index e49225400..239a70e1f 100644 --- a/roles/test-upload-logs-swift/README.rst +++ b/roles/test-upload-logs-swift/README.rst @@ -78,3 +78,17 @@ This uploads logs to an OpenStack Object Store (Swift) container. More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`. + +.. zuul:rolevar:: zuul_log_include_download_script + :default: False + + Generate a script from ``zuul_log_download_template`` in the root + directory of the uploaded logs to facilitate easy bulk download. + +.. zuul:rolevar:: zuul_log_download_template + :default: templates/download-logs.sh.j2 + + Path to template file if ``zuul_log_include_download_script`` is + set. See the sample file for parameters available to the template. + The file will be placed in the root of the uploaded logs (with + ``.j2`` suffix removed). diff --git a/roles/test-upload-logs-swift/defaults/main.yaml b/roles/test-upload-logs-swift/defaults/main.yaml index 27893357b..816d5212b 100644 --- a/roles/test-upload-logs-swift/defaults/main.yaml +++ b/roles/test-upload-logs-swift/defaults/main.yaml @@ -2,3 +2,5 @@ zuul_log_partition: false zuul_log_container: logs zuul_log_container_public: true zuul_log_create_indexes: true +zuul_log_include_download_script: true +zuul_log_download_template: '{{ role_path }}/templates/download-logs.sh.j2' \ No newline at end of file diff --git a/roles/test-upload-logs-swift/library/test-fixtures/download-logs-sample.sh b/roles/test-upload-logs-swift/library/test-fixtures/download-logs-sample.sh new file mode 100644 index 000000000..335d93936 --- /dev/null +++ b/roles/test-upload-logs-swift/library/test-fixtures/download-logs-sample.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Download all logs + +# +# To use this file +# +# curl "http://fakebaseurl.com/download-logs.sh" | bash +# +# Logs will be copied in a temporary directory as described in the +# output. Set DOWNLOAD_DIR to an empty directory if you wish to +# override this. +# + +BASE_URL=http://fakebaseurl.com + +function log { + echo "$(date -Iseconds) | $@" +} + +function save_file { + local file="$1" + + curl -s --compressed --create-dirs -o "${file}" "${BASE_URL}/${file}" + + # Using --compressed we will send an Accept-Encoding: gzip header + # and the data will come to us across the network compressed. + # However, sometimes things like OpenStack's log server will send + # .gz files (as stored on its disk) uncompressed, so we check if + # this really looks like an ASCII file and rename for clarity. + if [[ "${file}" == *.gz ]]; then + local type=$(file "${file}") + if [[ "${type}" =~ "ASCII text" ]] || [[ "${type}" =~ "Unicode text" ]]; then + local new_name=${file%.gz} + log "Renaming to ${new_name}" + mv "${file}" "${new_name}" + fi + fi + +} + +if [[ -z "${DOWNLOAD_DIR}" ]]; then + DOWNLOAD_DIR=$(mktemp -d --tmpdir zuul-logs.XXXXXX) +fi +log "Saving logs to ${DOWNLOAD_DIR}" + +pushd "${DOWNLOAD_DIR}" > /dev/null + + + +log "Getting ${BASE_URL}/job-output.json [ 0001 / 0010 ]" +save_file "job-output.json" + +log "Getting ${BASE_URL}/controller/compressed.gz [ 0002 / 0010 ]" +save_file "controller/compressed.gz" + +log "Getting ${BASE_URL}/controller/cpu-load.svg [ 0003 / 0010 ]" +save_file "controller/cpu-load.svg" + +log "Getting ${BASE_URL}/controller/journal.xz [ 0004 / 0010 ]" +save_file "controller/journal.xz" + +log "Getting ${BASE_URL}/controller/service_log.txt [ 0005 / 0010 ]" +save_file "controller/service_log.txt" + +log "Getting ${BASE_URL}/controller/syslog [ 0006 / 0010 ]" +save_file "controller/syslog" + +log "Getting ${BASE_URL}/controller/subdir/foo::3.txt [ 0007 / 0010 ]" +save_file "controller/subdir/foo::3.txt" + +log "Getting ${BASE_URL}/controller/subdir/subdir.txt [ 0008 / 0010 ]" +save_file "controller/subdir/subdir.txt" + +log "Getting ${BASE_URL}/zuul-info/inventory.yaml [ 0009 / 0010 ]" +save_file "zuul-info/inventory.yaml" + +log "Getting ${BASE_URL}/zuul-info/zuul-info.controller.txt [ 0010 / 0010 ]" +save_file "zuul-info/zuul-info.controller.txt" + + +popd >/dev/null + +log "Download complete!" \ No newline at end of file diff --git a/roles/test-upload-logs-swift/library/zuul_swift_upload.py b/roles/test-upload-logs-swift/library/zuul_swift_upload.py index c25720fc5..0e90f8053 100755 --- a/roles/test-upload-logs-swift/library/zuul_swift_upload.py +++ b/roles/test-upload-logs-swift/library/zuul_swift_upload.py @@ -30,6 +30,7 @@ import io import logging import mimetypes import os +import jinja2 try: import queue as queuelib except ImportError: @@ -54,6 +55,7 @@ import requests.exceptions import requestsexceptions import keystoneauth1.exceptions +from ansible.module_utils._text import to_text from ansible.module_utils.basic import AnsibleModule try: @@ -265,13 +267,15 @@ class FileDetail(): to push to swift. """ - def __init__(self, full_path, relative_path, filename=None): + def __init__(self, full_path, relative_path, + filename=None, is_index=False): """ Args: full_path (str): The absolute path to the file on disk. relative_path (str): The relative path from the artifacts source used for links. filename (str): An optional alternate filename in links. + is_index (bool): Is this file an index """ # Make FileNotFoundError exception to be compatible with python2 try: @@ -285,6 +289,7 @@ class FileDetail(): else: self.filename = filename self.relative_path = relative_path + self.is_index = is_index if self.full_path and os.path.isfile(self.full_path): mime_guess, encoding = mimetypes.guess_type(self.full_path) @@ -305,7 +310,8 @@ class FileDetail(): def __repr__(self): t = 'Folder' if self.folder else 'File' - return '<%s %s>' % (t, self.relative_path) + return '<%s %s%s>' % (t, self.relative_path, + ' (index)' if self.is_index else '') class FileList(Sequence): @@ -411,6 +417,7 @@ class Indexer(): FileList - make_indexes() : make index.html in folders + - make_download_script() : make a script to download all logs """ def __init__(self, file_list): ''' @@ -530,7 +537,8 @@ class Indexer(): if full_path: filename = os.path.basename(full_path) relative_name = os.path.join(folder, filename) - indexes[folder] = FileDetail(full_path, relative_name) + indexes[folder] = FileDetail(full_path, relative_name, + is_index=True) # This appends the index file at the end of the group of files # for each directory. @@ -553,6 +561,41 @@ class Indexer(): new_list.reverse() self.file_list.file_list = new_list + def make_download_script(self, base_url, download_template): + '''Make a download script from template + + Note since you need the base_url, it really only makes sense + to call this after the Uploader() is initalised. + + Args: + base_url (str): The base URL to prefix + download_template (str): Path to a jinja2 template + + Return: + None; a file with the same name as the template (stripped of + .j2 if present) is added to self.file_list for upload. + ''' + # Prune the list to just be files, no indexes (this should run + # before indexing anyway) + download_files = [f for f in self.file_list + if not f.folder and not f.is_index] + output_filename = os.path.basename(download_template[:-3] + if download_template.endswith('.j2') + else download_template) + output = os.path.join(self.file_list.get_tempdir(), output_filename) + + with open(download_template) as f, open(output, 'wb') as output: + logging.debug("Writing template %s" % output.name) + template = jinja2.Template(f.read()) + rendered = template.stream( + base_url=base_url.rstrip('/'), + # jinja wants unicode input + file_list=[to_text(f.relative_path) for f in download_files]) + rendered.dump(output, encoding='utf-8') + + download_script = FileDetail(output.name, output_filename) + self.file_list.file_list.append(download_script) + class GzipFilter(): chunk_size = 16384 @@ -604,7 +647,13 @@ class DeflateFilter(): class Uploader(): def __init__(self, cloud, container, prefix=None, delete_after=None, - public=True): + public=True, dry_run=False): + + if dry_run: + self.dry_run = True + self.url = 'http://dry-run-url.com/a/path/' + return + self.cloud = cloud self.container = container self.prefix = prefix or '' @@ -670,6 +719,10 @@ class Uploader(): def upload(self, file_list): """Spin up thread pool to upload to swift""" + + if self.dry_run: + return + num_threads = min(len(file_list), MAX_UPLOAD_THREADS) threads = [] queue = queuelib.Queue() @@ -753,7 +806,7 @@ class Uploader(): def run(cloud, container, files, indexes=True, parent_links=True, topdir_parent_link=False, partition=False, footer='index_footer.html', delete_after=15552000, - prefix=None, public=True, dry_run=False): + prefix=None, public=True, dry_run=False, download_template=''): if prefix: prefix = prefix.lstrip('/') @@ -769,8 +822,16 @@ def run(cloud, container, files, for file_path in files: file_list.add(file_path) + # Upload. + uploader = Uploader(cloud, container, prefix, delete_after, + public, dry_run) + indexer = Indexer(file_list) + # (Possibly) make download script + if download_template: + indexer.make_download_script(uploader.url, download_template) + # (Possibly) make indexes. if indexes: indexer.make_indexes(create_parent_links=parent_links, @@ -781,14 +842,6 @@ def run(cloud, container, files, for x in file_list: logging.debug(x) - # Do no connect to swift or do any uploading in a dry run - if dry_run: - # No URL is known, so return nothing - return - - # Upload. - uploader = Uploader(cloud, container, prefix, delete_after, - public) uploader.upload(file_list) return uploader.url @@ -807,6 +860,7 @@ def ansible_main(): footer=dict(type='str'), delete_after=dict(type='int'), prefix=dict(type='str'), + download_template=dict(type='str'), ) ) @@ -821,7 +875,8 @@ def ansible_main(): footer=p.get('footer'), delete_after=p.get('delete_after', 15552000), prefix=p.get('prefix'), - public=p.get('public')) + public=p.get('public'), + download_template=p.get('download_template')) except (keystoneauth1.exceptions.http.HttpError, requests.exceptions.RequestException): s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name) @@ -863,6 +918,9 @@ def cli_main(): 'upload. Default is 6 months (15552000 seconds) ' 'and if set to 0 X-Delete-After will not be set', type=int) + parser.add_argument('--download-template', default='', + help='Path to a Jinja2 template that will be filled ' + 'out to create an automatic download script') parser.add_argument('--prefix', help='Prepend this path to the object names when ' 'uploading') @@ -900,7 +958,8 @@ def cli_main(): delete_after=args.delete_after, prefix=args.prefix, public=not args.no_public, - dry_run=args.dry_run) + dry_run=args.dry_run, + download_template=args.download_template) print(url) diff --git a/roles/test-upload-logs-swift/tasks/main.yaml b/roles/test-upload-logs-swift/tasks/main.yaml index eba0d2a3a..4dc6c5a9e 100644 --- a/roles/test-upload-logs-swift/tasks/main.yaml +++ b/roles/test-upload-logs-swift/tasks/main.yaml @@ -16,6 +16,12 @@ tags: - skip_ansible_lint + - name: Set download template + set_fact: + download_template: "{{ zuul_log_download_template }}" + when: + - zuul_log_include_download_script + - name: Upload logs to swift delegate_to: localhost zuul_swift_upload: @@ -28,6 +34,7 @@ files: - "{{ zuul.executor.log_root }}/" delete_after: "{{ zuul_log_delete_after | default(omit) }}" + download_template: "{{ download_template | default(omit) }}" register: upload_results - name: Return log URL to Zuul