opendev/system-config
Code Issues Proposed changes

Separate out index generation for swift files

Browse Source 
This will allow us to arbitrarily pass in lists of files and generate
indexes for them in JJB macros.

Change-Id: Ied3bd3161a2cd75f63bb17731e5d3af9d6d8a630
This commit is contained in:
Joshua Hesketh 2014-07-16 20:23:56 +10:00
parent bd9947a1d4
commit d5411ce2b6
1 changed files with 94 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
modules/openstack_project/files/slave_scripts/zuul_swift_upload.py
View File

@ -23,7 +23,9 @@ import argparse
import magic import magic
import os import os
import requests import requests
import sys
import tempfile import tempfile
import time
def generate_log_index(file_list, logserver_prefix, swift_destination_prefix): def generate_log_index(file_list, logserver_prefix, swift_destination_prefix):
@ -85,19 +87,23 @@ def swift_form_post_submit(file_list, url, hmac_body, signature):
payload['expires']) = hmac_body.split('\n') payload['expires']) = hmac_body.split('\n')
payload['signature'] = signature payload['signature'] = signature
if len(file_list) > payload['max_file_count']: # Loop over the file list in chunks of max_file_count
# We can't upload this many files! We'll do what we can but the job for sub_file_list in (file_list[pos:pos + int(payload['max_file_count'])]
# should be reconfigured for pos in xrange(0, len(file_list),
file_list = file_list[:payload['max_file_count']] int(payload['max_file_count']))):
if payload['expires'] < time.time():
raise Exception("Ran out of time uploading files!")
files = {} files = {}
# Zuul's log path is generated without a tailing slash. As such the # Zuul's log path is generated without a tailing slash. As such the
# object prefix does not contain a slash and the files would be # object prefix does not contain a slash and the files would be
# uploaded as 'prefix' + 'filename'. Assume we want the destination # uploaded as 'prefix' + 'filename'. Assume we want the destination
# url to look like a folder and make sure there's a slash between. # url to look like a folder and make sure there's a slash between.
filename_prefix = '/' if url[-1] != '/' else '' filename_prefix = '/' if url[-1] != '/' else ''
for i, f in enumerate(file_list): for i, f in enumerate(sub_file_list):
if os.path.getsize(f['path']) > int(payload['max_file_size']):
sys.stderr.write('Warning: %s exceeds %d bytes. Skipping...\n'
% (f['path'], int(payload['max_file_size'])))
continue
files['file%d' % (i + 1)] = (filename_prefix + f['filename'], files['file%d' % (i + 1)] = (filename_prefix + f['filename'],
open(f['path'], 'rb'), open(f['path'], 'rb'),
get_file_mime(f['path'])) get_file_mime(f['path']))
@ -105,8 +111,8 @@ def swift_form_post_submit(file_list, url, hmac_body, signature):
requests.post(url, data=payload, files=files) requests.post(url, data=payload, files=files)
def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature, def build_file_list(file_path, logserver_prefix, swift_destination_prefix,
logserver_prefix, swift_destination_prefix): create_dir_indexes=True):
"""Upload to swift using instructions from zuul""" """Upload to swift using instructions from zuul"""
# file_list: a list of dicts with {path=..., filename=...} where filename # file_list: a list of dicts with {path=..., filename=...} where filename
@ -115,24 +121,36 @@ def zuul_swift_upload(file_path, swift_url, swift_hmac_body, swift_signature,
if os.path.isfile(file_path): if os.path.isfile(file_path):
file_list.append({'filename': os.path.basename(file_path), file_list.append({'filename': os.path.basename(file_path),
'path': file_path}) 'path': file_path})
index_file = file_path
elif os.path.isdir(file_path): elif os.path.isdir(file_path):
if file_path[-1] == os.sep:
file_path = file_path[:-1]
parent_dir = os.path.dirname(file_path)
for path, folders, files in os.walk(file_path): for path, folders, files in os.walk(file_path):
folder_contents = []
for f in files: for f in files:
full_path = os.path.join(path, f) full_path = os.path.join(path, f)
relative_name = os.path.relpath(full_path, file_path) relative_name = os.path.relpath(full_path, parent_dir)
file_list.append({'filename': relative_name, push_file = {'filename': relative_name,
'path': full_path}
folder_contents.append(push_file)
file_list.append(push_file)
for f in folders:
full_path = os.path.join(path, f) + '/'
relative_name = os.path.relpath(full_path, parent_dir)
folder_contents.append({'filename': relative_name,
'path': full_path}) 'path': full_path})
index_file = make_index_file(file_list, logserver_prefix,
if create_dir_indexes:
index_file = make_index_file(folder_contents, logserver_prefix,
swift_destination_prefix) swift_destination_prefix)
file_list.append({'filename': os.path.basename(index_file), relative_name = os.path.relpath(path, parent_dir)
file_list.append({
'filename': os.path.join(relative_name,
os.path.basename(index_file)),
'path': index_file}) 'path': index_file})
swift_form_post_submit(file_list, swift_url, swift_hmac_body, return file_list
swift_signature)
return os.path.join(logserver_prefix, swift_destination_prefix,
os.path.basename(index_file))
def grab_args(): def grab_args():
@ -140,24 +158,57 @@ def grab_args():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Upload results to swift using instructions from zuul" description="Upload results to swift using instructions from zuul"
) )
parser.add_argument('--no-indexes', action='store_true',
help='do not generate any indexes at all')
parser.add_argument('--no-root-index', action='store_true',
help='do not generate a root index')
parser.add_argument('--no-dir-indexes', action='store_true',
help='do not generate a indexes inside dirs')
parser.add_argument('-n', '--name', default="logs", parser.add_argument('-n', '--name', default="logs",
help='The instruction-set to use') help='The instruction-set to use')
parser.add_argument('files', nargs='+', help='the file(s) to upload') parser.add_argument('files', nargs='+', help='the file(s) to upload')
return parser.parse_args() return parser.parse_args()
if __name__ == '__main__': if __name__ == '__main__':
args = grab_args() args = grab_args()
for file_path in args.files: file_list = []
root_list = []
try: try:
result_url = zuul_swift_upload( logserver_prefix = os.environ['SWIFT_%s_LOGSERVER_PREFIX' % args.name]
file_path, swift_destination_prefix = os.environ['LOG_PATH']
os.environ['SWIFT_%s_URL' % args.name], swift_url = os.environ['SWIFT_%s_URL' % args.name]
os.environ['SWIFT_%s_HMAC_BODY' % args.name], swift_hmac_body = os.environ['SWIFT_%s_HMAC_BODY' % args.name]
os.environ['SWIFT_%s_SIGNATURE' % args.name], swift_signature = os.environ['SWIFT_%s_SIGNATURE' % args.name]
os.environ['SWIFT_%s_LOGSERVER_PREFIX' % args.name],
os.environ['LOG_PATH']
)
print result_url
except KeyError as e: except KeyError as e:
print 'Environment variable %s not found' % e print 'Environment variable %s not found' % e
quit()
for file_path in args.files:
if os.path.isfile(file_path):
root_list.append({'filename': os.path.basename(file_path),
'path': file_path})
else:
root_list.append({'filename': os.path.basename(file_path) + '/',
'path': file_path})
file_list += build_file_list(
file_path, logserver_prefix, swift_destination_prefix,
(not (args.no_indexes and args.no_dir_indexes))
)
index_file = ''
if not (args.no_indexes and args.no_root_index):
index_file = make_index_file(root_list, logserver_prefix,
swift_destination_prefix)
file_list.append({
'filename': os.path.basename(index_file),
'path': index_file})
swift_form_post_submit(file_list, swift_url, swift_hmac_body,
swift_signature)
print os.path.join(logserver_prefix, swift_destination_prefix,
os.path.basename(index_file))