Change log paths to match zuuls current deterministic structure

etc/turbo-hipster/config.json

@@ -12,7 +12,7 @@
         {
             "name": "gate_real_db_upgrade",
             "datasets_dir": "/var/lib/turbo-hipster/datasets",
-            "gate": "build:gate-real-db-upgrade_nova_mysql"
+            "job": "gate-real-db-upgrade_nova_mysql"
         }
     ],
     "publish_logs": {

tests/etc/config.json

@@ -12,7 +12,7 @@
         {
             "name": "gate_real_db_upgrade",
             "datasets_dir": "/home/josh/var/lib/turbo-hipster/datasets",
-            "gate": "build:gate-real-db-upgrade_nova_mysql"
+            "job": "gate-real-db-upgrade_nova_mysql"
         }
     ],
     "publish_logs":

turbo_hipster/lib/utils.py

@@ -186,17 +186,17 @@ def execute_to_log(cmd, logfile, timeout=-1,
     log_hanlder.close()
 
 
-def push_file(job_unique_number, file_path, publish_config):
+def push_file(dest_dir, file_path, publish_config):
     """ Push a log file to a server. Returns the public URL """
     method = publish_config['type'] + '_push_file'
     if method in globals() and hasattr(globals()[method], '__call__'):
-        return globals()[method](job_unique_number, file_path, publish_config)
+        return globals()[method](dest_dir, file_path, publish_config)
 
 
-def swift_push_file(job_unique_number, file_path, swift_config):
+def swift_push_file(dest_dir, file_path, swift_config):
     """ Push a log file to a swift server. """
     with open(file_path, 'r') as fd:
-        name = job_unique_number + '_' + os.path.basename(file_path)
+        name = dest_dir + '_' + os.path.basename(file_path)
         con = swiftclient.client.Connection(swift_config['authurl'],
                                             swift_config['user'],
                                             swift_config['apikey'])
@@ -204,9 +204,9 @@ def swift_push_file(job_unique_number, file_path, swift_config):
         return obj
 
 
-def local_push_file(job_unique_number, file_path, local_config):
+def local_push_file(dest_dir, file_path, local_config):
     """ Copy the file locally somewhere sensible """
-    dest_dir = os.path.join(local_config['path'], job_unique_number)
+    dest_dir = os.path.join(local_config['path'], dest_dir)
     dest_filename = os.path.basename(file_path)
     if not os.path.isdir(dest_dir):
         os.makedirs(dest_dir)
@@ -214,10 +214,18 @@ def local_push_file(job_unique_number, file_path, local_config):
     dest_file = os.path.join(dest_dir, dest_filename)
 
     shutil.copyfile(file_path, dest_file)
-    return local_config['prepend_url'] + os.path.join(job_unique_number,
-                                                      dest_filename)
+    return local_config['prepend_url'] + os.path.join(dest_dir, dest_filename)
 
 
-def scp_push_file(job_unique_number, file_path, local_config):
+def scp_push_file(dest_dir, file_path, local_config):
     """ Copy the file remotely over ssh """
     pass
+
+
+def determine_job_identifier(zuul_arguments, job, unique):
+    return os.path.join(zuul_arguments['ZUUL_CHANGE'][:2],
+                        zuul_arguments['ZUUL_CHANGE'],
+                        zuul_arguments['ZUUL_PATCHSET'],
+                        zuul_arguments['ZUUL_PIPELINE'],
+                        job,
+                        unique[:7])

turbo_hipster/task_plugins/gate_real_db_upgrade/handle_results.py

@@ -52,18 +52,17 @@ def make_index_file(datasets, index_filename):
     return os.path.join(tempdir, index_filename)
 
 
-def generate_push_results(datasets, job_unique_number, publish_config):
+def generate_push_results(datasets, publish_config):
     """ Generates and pushes results """
 
     for i, dataset in enumerate(datasets):
-        result_uri = push_file(job_unique_number,
+        result_uri = push_file(dataset['determined_path'],
                                dataset['log_file_path'],
                                publish_config)
         datasets[i]['result_uri'] = result_uri
 
     index_file = make_index_file(datasets, 'index.html')
-    index_file_url = push_file(job_unique_number,
-                               index_file,
+    index_file_url = push_file(dataset['determined_path'], index_file,
                                publish_config)
 
     return index_file_url

turbo_hipster/task_plugins/gate_real_db_upgrade/task.py

@@ -72,7 +72,8 @@ class Runner(threading.Thread):
         self.register_functions()
 
     def register_functions(self):
-        self.gearman_worker.registerFunction(self.plugin_config['gate'])
+        self.gearman_worker.registerFunction(
+            'build:' + self.plugin_config['job'])
 
     def stop(self):
         self._stop.set()
@@ -157,7 +158,6 @@ class Runner(threading.Thread):
         self.log.debug("Process the resulting files (upload/push)")
         index_url = handle_results.generate_push_results(
             self.job_datasets,
-            self.job.unique,
             self.global_config['publish_logs']
         )
         self.log.debug("Index URL found at %s" % index_url)
@@ -213,9 +213,13 @@ class Runner(threading.Thread):
             if (self.job_arguments['ZUUL_PROJECT'] ==
                     dataset['config']['project'] and
                     self._get_project_command(dataset['config']['type'])):
+                dataset['determined_path'] = utils.determine_job_identifier(
+                    self.job_arguments, self.plugin_config['job'],
+                    self.job.unique
+                )
                 dataset['log_file_path'] = os.path.join(
                     self.global_config['jobs_working_dir'],
-                    self.job.unique,
+                    dataset['determined_path'],
                     dataset['name'] + '.log'
                 )
                 dataset['result'] = 'UNTESTED'