diff --git a/roles/emit-job-header/README.rst b/roles/emit-job-header/README.rst index bdb97c7ed..b67d66b62 100644 --- a/roles/emit-job-header/README.rst +++ b/roles/emit-job-header/README.rst @@ -5,3 +5,13 @@ Log a few lines about the job. .. zuul:rolevar:: zuul_log_url Base URL where logs are to be found. + +.. zuul:rolevar:: zuul_log_path_shard_build + :default: False + + This var is consumed by set-zuul-log-path-fact which emit-job-header + calls into. If you set this you will get log paths prefixed with the + first three characters of the build uuid. This will improve log file + sharding. + + More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build` diff --git a/roles/set-zuul-log-path-fact/README.rst b/roles/set-zuul-log-path-fact/README.rst index d64eec8af..33a36d1da 100644 --- a/roles/set-zuul-log-path-fact/README.rst +++ b/roles/set-zuul-log-path-fact/README.rst @@ -1 +1,12 @@ Sets a fact named ``zuul_log_path`` from zuul variables + +**Role Variables** + +.. zuul:rolevar:: zuul_log_path_shard_build + :type: bool + :default: False + + Flag to specify whether or not paths that include a three character + prefix based on the build uuid should prefix the log path. This is + particularly useful for object storage systems where we want to + spread out the number of files per container. diff --git a/roles/set-zuul-log-path-fact/defaults/main.yaml b/roles/set-zuul-log-path-fact/defaults/main.yaml new file mode 100644 index 000000000..b1abb3a6f --- /dev/null +++ b/roles/set-zuul-log-path-fact/defaults/main.yaml @@ -0,0 +1 @@ +zuul_log_path_shard_build: false diff --git a/roles/set-zuul-log-path-fact/tasks/main.yaml b/roles/set-zuul-log-path-fact/tasks/main.yaml index c49c3a85e..f9dff1b5a 100644 --- a/roles/set-zuul-log-path-fact/tasks/main.yaml +++ b/roles/set-zuul-log-path-fact/tasks/main.yaml @@ -1,14 +1,35 @@ -- name: Set log path for a change - when: zuul.change is defined - set_fact: - zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" +- name: Fileserver friendly log path specifications + when: not zuul_log_path_shard_build + block: + - name: Set log path for a change + when: zuul.change is defined + set_fact: + zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" -- name: Set log path for a ref update - when: zuul.newrev is defined - set_fact: - zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + - name: Set log path for a ref update + when: zuul.newrev is defined + set_fact: + zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" -- name: Set log path for a periodic job - when: zuul.change is not defined and zuul.newrev is not defined - set_fact: - zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + - name: Set log path for a periodic job + when: zuul.change is not defined and zuul.newrev is not defined + set_fact: + zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + +- name: object store friendly log path specifications + when: zuul_log_path_shard_build + block: + - name: Set log path for a change + when: zuul.change is defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + + - name: Set log path for a ref update + when: zuul.newrev is defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}" + + - name: Set log path for a periodic job + when: zuul.change is not defined and zuul.newrev is not defined + set_fact: + zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}" diff --git a/test-playbooks/base-roles/emit-job-header.yaml b/test-playbooks/base-roles/emit-job-header.yaml index 2dce32de2..51991e1e3 100644 --- a/test-playbooks/base-roles/emit-job-header.yaml +++ b/test-playbooks/base-roles/emit-job-header.yaml @@ -1,4 +1,29 @@ -- name: Test the emit-job-header role +- name: Test the emit-job-header role with swift + hosts: all + roles: + - role: emit-job-header + zuul_log_url: "http://logs.openstack.org" + zuul_log_path_shard_build: true + post_tasks: + # All emit-job-header does is a debug statement so the worst that would + # happen would be that the debug task would fail outright and we'd prevent + # something breaking that debug statement from merging just by running it. + # However, the emit-job-header role includes the set-zuul-log-path-fact + # role. We can only test for zuul_log_path against changes, though. + - name: Assert zuul_log_path by set-zuul-log-path-fact for a change + assert: + that: + - zuul_log_path is defined + - zuul.change in zuul_log_path + - zuul.patchset in zuul_log_path + - zuul.pipeline in zuul_log_path + - zuul.job in zuul_log_path + - zuul.build[:3] == zuul_log_path[:3] + +# Note that the order of these two plays is important. We want the second one +# to run to be the one the creates the correct log path for the currently +# running system. +- name: Test the emit-job-header role without swift hosts: all roles: - role: emit-job-header @@ -17,3 +42,4 @@ - zuul.patchset in zuul_log_path - zuul.pipeline in zuul_log_path - zuul.job in zuul_log_path + - zuul.build[:3] != zuul_log_path[:3]