opendev/system-config
Code Issues Proposed changes

Switch Zuul/JJB to use LOG_PATH

Browse Source 
This calculates the full LOG_PATH in the Zuul config and pases it
to Jenkins.  The new path is similar to the old but with the
substitution of a short ZUUL_UUID at the end instead of the Jenkins
build number in order to avoid collisions from multiple Jenkins
masters.

Periodic jobs add a node name to their log path to avoid collisions
from multiple masters.  Unfortunately, that value is not accessible
to the logstash worker.  This can be solved by having Zuul trigger
periodic jobs.

Add the ZUUL_REF to logstash as "build_ref".

Requires https://review.openstack.org/#/c/39130/

Change-Id: I40bad59e3ad8ed6b4706762ed8b833fd15c13b0d
This commit is contained in:
James E. Blair 2013-07-29 10:46:53 -07:00
parent 75e2eec220
commit 0ccf564937
8 changed files with 33 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
manifests/site.pp
View File

@ -311,7 +311,7 @@ node 'zuul.openstack.org' {
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}',
url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}',
sysadmins => hiera('sysadmins'),
statsd_host => 'graphite.openstack.org',
gearman_workers => [
@ -326,7 +326,7 @@ node 'zuul-dev.openstack.org' {
gerrit_server => 'review.openstack.org',
gerrit_user => 'zuul-dev',
zuul_ssh_private_key => hiera('zuul_dev_ssh_private_key_contents'),
url_pattern => 'http://logs.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}',
url_pattern => 'http://logs.openstack.org/{build.parameters[LOG_PATH]}',
sysadmins => hiera('sysadmins'),
statsd_host => 'graphite.openstack.org',
gearman_workers => [

34
modules/openstack_project/files/jenkins_job_builder/config/macros.yaml
View File

@ -193,10 +193,10 @@
builders:
- shell: |
#!/bin/sh
if test "$BASE_LOG_PATH" ; then
echo "Detailed logs: http://logs.openstack.org/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/"
if test "$LOG_PATH" ; then
echo "Detailed logs: http://logs.openstack.org/$LOG_PATH/"
else
echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$BUILD_NUMBER/"
echo "Detailed logs: http://logs.openstack.org/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER/"
fi
- builder:
@ -239,7 +239,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
copy-console: true
copy-after-failure: true
@ -249,7 +249,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
copy-console: true
copy-after-failure: true
@ -259,15 +259,15 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: 'logs/**'
keep-hierarchy: true
copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/testr_results.html.gz'
keep-hierarchy: false
copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/subunit_log.txt.gz'
keep-hierarchy: false
copy-after-failure: true
@ -278,15 +278,15 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: 'logs/**'
keep-hierarchy: true
copy-after-failure: true
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: '**/testr_results.html.gz'
keep-hierarchy: false
copy-after-failure: true
- target: 'logs/periodic/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/periodic/$JOB_NAME/$NODE_NAME/$BUILD_NUMBER'
source: '**/subunit_log.txt.gz'
keep-hierarchy: false
copy-after-failure: true
@ -309,7 +309,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: 'cover/**'
keep-hierarchy: true
copy-after-failure: true
@ -320,19 +320,19 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/*nose_results.html'
keep-hierarchy: false
copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/*testr_results.html.gz'
keep-hierarchy: false
copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '.testrepository/tmp*'
keep-hierarchy: false
copy-after-failure: true
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/*subunit_log.txt.gz'
keep-hierarchy: false
copy-after-failure: true
@ -343,7 +343,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/'
- target: 'docs-draft/$LOG_PATH'
source: 'doc/build/html/**'
keep-hierarchy: true
copy-after-failure: true

2
modules/openstack_project/files/jenkins_job_builder/config/manuals.yaml
View File

@ -19,7 +19,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'docs-draft/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER/ops/'
- target: 'docs-draft/$LOG_PATH/ops/'
source: 'doc/src/docbkx/openstack-ops/target/docbkx/webhelp/**'
keep-hierarchy: true
copy-after-failure: true

2
modules/openstack_project/files/jenkins_job_builder/config/pbr.yaml
View File

@ -35,7 +35,7 @@
- scp:
site: 'static.openstack.org'
files:
- target: 'logs/$BASE_LOG_PATH/$JOB_NAME/$BUILD_NUMBER'
- target: 'logs/$LOG_PATH'
source: '**/pip.log'
keep-hierarchy: false
copy-after-failure: true

14
modules/openstack_project/files/logstash/log-gearman-client.py
View File

@ -76,10 +76,7 @@ class EventProcessor(threading.Thread):
def _get_log_dir(self, event):
parameters = event["build"].get("parameters", {})
base = parameters.get('BASE_LOG_PATH', 'periodic')
base += '/{name}/{number}/'.format(
name=event.get("name", "UNKNOWN"),
number=event["build"].get("number", "UNKNOWN"))
base = parameters.get('LOG_PATH', 'UNKNOWN')
return base
def _parse_fields(self, event, filename):
@ -90,13 +87,14 @@ class EventProcessor(threading.Thread):
fields["build_number"] = event["build"].get("number", "UNKNOWN")
parameters = event["build"].get("parameters", {})
fields["build_queue"] = parameters.get("ZUUL_PIPELINE", "UNKNOWN")
if fields["build_queue"] in ["check", "gate"]:
fields["build_ref"] = parameters.get("ZUUL_REF", "UNKNOWN")
if parameters.get("ZUUL_CHANGE"):
fields["build_change"] = parameters.get("ZUUL_CHANGE", "UNKNOWN")
fields["build_patchset"] = parameters.get("ZUUL_PATCHSET",
"UNKNOWN")
elif fields["build_queue"] in ["post", "pre-release", "release"]:
fields["build_shortref"] = parameters.get("ZUUL_SHORT_NEWREV",
"UNKNOWN")
elif parameters.get("ZUUL_NEWREV"):
fields["build_newrev"] = parameters.get("ZUUL_NEWREV",
"UNKNOWN")
return fields
def _parse_event(self, event, fileopts):

4
modules/openstack_project/files/zuul/layout-dev.yaml
View File

@ -58,9 +58,9 @@ jobs:
- name: openstack-ha-guide
branch: ^master$
- name: gate-openstack-cli-guide
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{build.number}/cli/quick-start
success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/cli/quick-start
- name: ^gate-.*-docs$
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/
success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/
- name: ^gate-.*-requirements$
failure-message: Incompatible requirement found; see https://wiki.openstack.org/wiki/Requirements
branch: ^(?!stable/(folsom|grizzly)).*$

2
modules/openstack_project/files/zuul/layout.yaml
View File

@ -141,7 +141,7 @@ jobs:
- name: openstack-security-guide
branch: ^master$
- name: ^gate-.*-docs$
success-pattern: http://docs-draft.openstack.org/{build.parameters[BASE_LOG_PATH]}/{job.name}/{build.number}/doc/build/html/
success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/doc/build/html/
- name: infra-publications-publish
branch: ^(?!master).*$
- name: ^gate-.*-requirements$

4
modules/openstack_project/files/zuul/openstack_functions.py
View File

@ -1,4 +1,4 @@
def set_log_url(item, params):
def set_log_url(item, job, params):
if hasattr(item.change, 'refspec'):
path = "%s/%s/%s/%s" % (
params['ZUUL_CHANGE'][-2:], params['ZUUL_CHANGE'],
@ -8,3 +8,5 @@ def set_log_url(item, params):
params['ZUUL_NEWREV'][:2], params['ZUUL_NEWREV'],
params['ZUUL_PIPELINE'])
params['BASE_LOG_PATH'] = path
params['LOG_PATH'] = path + '/%s/%s' % (job.name,
params['ZUUL_UUID'][:7])