From 0ccf5649374441232bddc60498d8fe57d0ef2d1b Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 29 Jul 2013 10:46:53 -0700 Subject: [PATCH] Switch Zuul/JJB to use LOG_PATH This calculates the full LOG_PATH in the Zuul config and pases it to Jenkins. The new path is similar to the old but with the substitution of a short ZUUL_UUID at the end instead of the Jenkins build number in order to avoid collisions from multiple Jenkins masters. Periodic jobs add a node name to their log path to avoid collisions from multiple masters. Unfortunately, that value is not accessible to the logstash worker. This can be solved by having Zuul trigger periodic jobs. Add the ZUUL_REF to logstash as "build_ref". Requires https://review.openstack.org/#/c/39130/ Change-Id: I40bad59e3ad8ed6b4706762ed8b833fd15c13b0d --- manifests/site.pp | 4 +-- .../jenkins_job_builder/config/macros.yaml | 34 +++++++++---------- .../jenkins_job_builder/config/manuals.yaml | 2 +- .../files/jenkins_job_builder/config/pbr.yaml | 2 +- .../files/logstash/log-gearman-client.py | 14 ++++---- .../files/zuul/layout-dev.yaml | 4 +-- .../openstack_project/files/zuul/layout.yaml | 2 +- .../files/zuul/openstack_functions.py | 4 ++- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/manifests/site.pp b/manifests/site.pp index e19bcb75a6..c95f35989f 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -311,7 +311,7 @@ node 'zuul.openstack.org' { gerrit_server => 'review.openstack.org', gerrit_user => 'jenkins', zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), - url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}', + url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}', sysadmins => hiera('sysadmins'), statsd_host => 'graphite.openstack.org', gearman_workers => [ @@ -326,7 +326,7 @@ node 'zuul-dev.openstack.org' { gerrit_server => 'review.openstack.org', gerrit_user => 'zuul-dev', zuul_ssh_private_key => hiera('zuul_dev_ssh_private_key_contents'), - url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}', + url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}', sysadmins => hiera('sysadmins'), statsd_host => 'graphite.openstack.org', gearman_workers => [ diff --git a/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml b/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml index 76665a8a6b..2b24464e2b 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/macros.yaml @@ -193,10 +193,10 @@ builders: - shell: | #!/bin/sh - if test "$BASE_LOG_PATH" ; then - echo "Detailed logs: http://logs.openstack.org/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/" + if test "$LOG_PATH" ; then + echo "Detailed logs: http://logs.openstack.org/$LOG_PATH/" else - echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$BUILD_NUMBER/" + echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER/" fi - builder: @@ -239,7 +239,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' copy-console: true copy-after-failure: true @@ -249,7 +249,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER' copy-console: true copy-after-failure: true @@ -259,15 +259,15 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: 'logs/**' keep-hierarchy: true copy-after-failure: true - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/testr_results.html.gz' keep-hierarchy: false copy-after-failure: true - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/subunit_log.txt.gz' keep-hierarchy: false copy-after-failure: true @@ -278,15 +278,15 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER' source: 'logs/**' keep-hierarchy: true copy-after-failure: true - - target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER' source: '**/testr_results.html.gz' keep-hierarchy: false copy-after-failure: true - - target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER' source: '**/subunit_log.txt.gz' keep-hierarchy: false copy-after-failure: true @@ -309,7 +309,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: 'cover/**' keep-hierarchy: true copy-after-failure: true @@ -320,19 +320,19 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/*nose_results.html' keep-hierarchy: false copy-after-failure: true - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/*testr_results.html.gz' keep-hierarchy: false copy-after-failure: true - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '.testrepository/tmp*' keep-hierarchy: false copy-after-failure: true - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/*subunit_log.txt.gz' keep-hierarchy: false copy-after-failure: true @@ -343,7 +343,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/' + - target: 'docs-draft/$LOG_PATH' source: 'doc/build/html/**' keep-hierarchy: true copy-after-failure: true diff --git a/modules/openstack_project/files/jenkins_job_builder/config/manuals.yaml b/modules/openstack_project/files/jenkins_job_builder/config/manuals.yaml index 93fc5230be..8c54477b20 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/manuals.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/manuals.yaml @@ -19,7 +19,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/ops/' + - target: 'docs-draft/$LOG_PATH/ops/' source: 'doc/src/docbkx/openstack-ops/target/docbkx/webhelp/**' keep-hierarchy: true copy-after-failure: true diff --git a/modules/openstack_project/files/jenkins_job_builder/config/pbr.yaml b/modules/openstack_project/files/jenkins_job_builder/config/pbr.yaml index e3db87e7d3..340c0e2f7f 100644 --- a/modules/openstack_project/files/jenkins_job_builder/config/pbr.yaml +++ b/modules/openstack_project/files/jenkins_job_builder/config/pbr.yaml @@ -35,7 +35,7 @@ - scp: site: 'static.openstack.org' files: - - target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER' + - target: 'logs/$LOG_PATH' source: '**/pip.log' keep-hierarchy: false copy-after-failure: true diff --git a/modules/openstack_project/files/logstash/log-gearman-client.py b/modules/openstack_project/files/logstash/log-gearman-client.py index 5ac7a32b8e..4816b94ac7 100644 --- a/modules/openstack_project/files/logstash/log-gearman-client.py +++ b/modules/openstack_project/files/logstash/log-gearman-client.py @@ -76,10 +76,7 @@ class EventProcessor(threading.Thread): def _get_log_dir(self, event): parameters = event["build"].get("parameters", {}) - base = parameters.get('BASE_LOG_PATH', 'periodic') - base += '/{name}/{number}/'.format( - name=event.get("name", "UNKNOWN"), - number=event["build"].get("number", "UNKNOWN")) + base = parameters.get('LOG_PATH', 'UNKNOWN') return base def _parse_fields(self, event, filename): @@ -90,13 +87,14 @@ class EventProcessor(threading.Thread): fields["build_number"] = event["build"].get("number", "UNKNOWN") parameters = event["build"].get("parameters", {}) fields["build_queue"] = parameters.get("ZUUL_PIPELINE", "UNKNOWN") - if fields["build_queue"] in ["check", "gate"]: + fields["build_ref"] = parameters.get("ZUUL_REF", "UNKNOWN") + if parameters.get("ZUUL_CHANGE"): fields["build_change"] = parameters.get("ZUUL_CHANGE", "UNKNOWN") fields["build_patchset"] = parameters.get("ZUUL_PATCHSET", "UNKNOWN") - elif fields["build_queue"] in ["post", "pre-release", "release"]: - fields["build_shortref"] = parameters.get("ZUUL_SHORT_NEWREV", - "UNKNOWN") + elif parameters.get("ZUUL_NEWREV"): + fields["build_newrev"] = parameters.get("ZUUL_NEWREV", + "UNKNOWN") return fields def _parse_event(self, event, fileopts): diff --git a/modules/openstack_project/files/zuul/layout-dev.yaml b/modules/openstack_project/files/zuul/layout-dev.yaml index b262e4517f..df993ef5f9 100644 --- a/modules/openstack_project/files/zuul/layout-dev.yaml +++ b/modules/openstack_project/files/zuul/layout-dev.yaml @@ -58,9 +58,9 @@ jobs: - name: openstack-ha-guide branch: ^master$ - name: gate-openstack-cli-guide - success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{build.number}/cli/quick-start + success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/cli/quick-start - name: ^gate-.*-docs$ - success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/ + success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/ - name: ^gate-.*-requirements$ failure-message: Incompatible requirement found; see https://wiki.openstack.org/wiki/Requirements branch: ^(?!stable/(folsom|grizzly)).*$ diff --git a/modules/openstack_project/files/zuul/layout.yaml b/modules/openstack_project/files/zuul/layout.yaml index 8031900d63..1870949d5b 100644 --- a/modules/openstack_project/files/zuul/layout.yaml +++ b/modules/openstack_project/files/zuul/layout.yaml @@ -141,7 +141,7 @@ jobs: - name: openstack-security-guide branch: ^master$ - name: ^gate-.*-docs$ - success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/ + success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/ - name: infra-publications-publish branch: ^(?!master).*$ - name: ^gate-.*-requirements$ diff --git a/modules/openstack_project/files/zuul/openstack_functions.py b/modules/openstack_project/files/zuul/openstack_functions.py index a586dbc6ec..c30188415a 100644 --- a/modules/openstack_project/files/zuul/openstack_functions.py +++ b/modules/openstack_project/files/zuul/openstack_functions.py @@ -1,4 +1,4 @@ -def set_log_url(item, params): +def set_log_url(item, job, params): if hasattr(item.change, 'refspec'): path = "%s/%s/%s/%s" % ( params['ZUUL_CHANGE'][-2:], params['ZUUL_CHANGE'], @@ -8,3 +8,5 @@ def set_log_url(item, params): params['ZUUL_NEWREV'][:2], params['ZUUL_NEWREV'], params['ZUUL_PIPELINE']) params['BASE_LOG_PATH'] = path + params['LOG_PATH'] = path + '/%s/%s' % (job.name, + params['ZUUL_UUID'][:7])