diff --git a/etc/turbo-hipster/config.json b/etc/turbo-hipster/config.json index b3307f3..2afd604 100644 --- a/etc/turbo-hipster/config.json +++ b/etc/turbo-hipster/config.json @@ -12,7 +12,7 @@ { "name": "gate_real_db_upgrade", "datasets_dir": "/var/lib/turbo-hipster/datasets", - "gate": "build:gate-real-db-upgrade_nova_mysql" + "job": "gate-real-db-upgrade_nova_mysql" } ], "publish_logs": { diff --git a/tests/etc/config.json b/tests/etc/config.json index b146205..587b110 100644 --- a/tests/etc/config.json +++ b/tests/etc/config.json @@ -12,7 +12,7 @@ { "name": "gate_real_db_upgrade", "datasets_dir": "/home/josh/var/lib/turbo-hipster/datasets", - "gate": "build:gate-real-db-upgrade_nova_mysql" + "job": "gate-real-db-upgrade_nova_mysql" } ], "publish_logs": diff --git a/turbo_hipster/lib/utils.py b/turbo_hipster/lib/utils.py index 71286d7..50f2a63 100644 --- a/turbo_hipster/lib/utils.py +++ b/turbo_hipster/lib/utils.py @@ -186,17 +186,17 @@ def execute_to_log(cmd, logfile, timeout=-1, log_hanlder.close() -def push_file(job_unique_number, file_path, publish_config): +def push_file(dest_dir, file_path, publish_config): """ Push a log file to a server. Returns the public URL """ method = publish_config['type'] + '_push_file' if method in globals() and hasattr(globals()[method], '__call__'): - return globals()[method](job_unique_number, file_path, publish_config) + return globals()[method](dest_dir, file_path, publish_config) -def swift_push_file(job_unique_number, file_path, swift_config): +def swift_push_file(dest_dir, file_path, swift_config): """ Push a log file to a swift server. """ with open(file_path, 'r') as fd: - name = job_unique_number + '_' + os.path.basename(file_path) + name = dest_dir + '_' + os.path.basename(file_path) con = swiftclient.client.Connection(swift_config['authurl'], swift_config['user'], swift_config['apikey']) @@ -204,9 +204,9 @@ def swift_push_file(job_unique_number, file_path, swift_config): return obj -def local_push_file(job_unique_number, file_path, local_config): +def local_push_file(dest_dir, file_path, local_config): """ Copy the file locally somewhere sensible """ - dest_dir = os.path.join(local_config['path'], job_unique_number) + dest_dir = os.path.join(local_config['path'], dest_dir) dest_filename = os.path.basename(file_path) if not os.path.isdir(dest_dir): os.makedirs(dest_dir) @@ -214,10 +214,18 @@ def local_push_file(job_unique_number, file_path, local_config): dest_file = os.path.join(dest_dir, dest_filename) shutil.copyfile(file_path, dest_file) - return local_config['prepend_url'] + os.path.join(job_unique_number, - dest_filename) + return local_config['prepend_url'] + os.path.join(dest_dir, dest_filename) -def scp_push_file(job_unique_number, file_path, local_config): +def scp_push_file(dest_dir, file_path, local_config): """ Copy the file remotely over ssh """ pass + + +def determine_job_identifier(zuul_arguments, job, unique): + return os.path.join(zuul_arguments['ZUUL_CHANGE'][:2], + zuul_arguments['ZUUL_CHANGE'], + zuul_arguments['ZUUL_PATCHSET'], + zuul_arguments['ZUUL_PIPELINE'], + job, + unique[:7]) diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py b/turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py index b3e7961..5eac355 100644 --- a/turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py +++ b/turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py @@ -52,18 +52,17 @@ def make_index_file(datasets, index_filename): return os.path.join(tempdir, index_filename) -def generate_push_results(datasets, job_unique_number, publish_config): +def generate_push_results(datasets, publish_config): """ Generates and pushes results """ for i, dataset in enumerate(datasets): - result_uri = push_file(job_unique_number, + result_uri = push_file(dataset['determined_path'], dataset['log_file_path'], publish_config) datasets[i]['result_uri'] = result_uri index_file = make_index_file(datasets, 'index.html') - index_file_url = push_file(job_unique_number, - index_file, + index_file_url = push_file(dataset['determined_path'], index_file, publish_config) return index_file_url diff --git a/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py b/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py index 113bdea..bc95ea1 100644 --- a/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py +++ b/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py @@ -72,7 +72,8 @@ class Runner(threading.Thread): self.register_functions() def register_functions(self): - self.gearman_worker.registerFunction(self.plugin_config['gate']) + self.gearman_worker.registerFunction( + 'build:' + self.plugin_config['job']) def stop(self): self._stop.set() @@ -157,7 +158,6 @@ class Runner(threading.Thread): self.log.debug("Process the resulting files (upload/push)") index_url = handle_results.generate_push_results( self.job_datasets, - self.job.unique, self.global_config['publish_logs'] ) self.log.debug("Index URL found at %s" % index_url) @@ -213,9 +213,13 @@ class Runner(threading.Thread): if (self.job_arguments['ZUUL_PROJECT'] == dataset['config']['project'] and self._get_project_command(dataset['config']['type'])): + dataset['determined_path'] = utils.determine_job_identifier( + self.job_arguments, self.plugin_config['job'], + self.job.unique + ) dataset['log_file_path'] = os.path.join( self.global_config['jobs_working_dir'], - self.job.unique, + dataset['determined_path'], dataset['name'] + '.log' ) dataset['result'] = 'UNTESTED'