opendev/system-config
Code Issues Proposed changes

production-playbook logs : move to post-run step

Browse Source 
If the production playbook times out, we don't get any logs collected
with the run.  By moving the log collection into a post-run step, we
should always get something copied to help us diagnose what is going
wrong.

Change-Id: I3e99b80e442db0cc87f8e8c9728b7697a5e4d1d3
This commit is contained in:
Ian Wienand 2022-07-14 16:32:58 +10:00
parent 3ab81bb834
commit 21efe11eed
3 changed files with 87 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
playbooks/zuul/run-production-playbook-post.yaml Normal file
View File

@ -0,0 +1,86 @@
- hosts: bridge.openstack.org
tasks:
- name: Encrypt log
when: infra_prod_playbook_encrypt_log|default(False)
block:
- name: Create temporary staging area for encrypted logs
tempfile:
state: directory
register: _encrypt_tempdir
- name: Copy log to tempdir as Zuul user
copy:
src: '/var/log/ansible/{{ playbook_name }}.log'
dest: '{{ _encrypt_tempdir.path }}'
owner: zuul
group: zuul
mode: '0644'
remote_src: yes
become: yes
- name: Encrypt logs
include_role:
name: encrypt-logs
vars:
encrypt_logs_files:
- '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
# Artifact URL should just point to root directory, so blank
encrypt_logs_artifact_path: ''
encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'
- name: Return logs
synchronize:
src: '{{ item[0] }}'
dest: '{{ item[1] }}'
mode: pull
verify_host: true
loop:
- ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
- ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']
always:
- name: Remove temporary staging
file:
path: '{{ _encrypt_tempdir.path }}'
state: absent
when: _encrypt_tempdir is defined
# Not using normal zuul job roles as bridge.openstack.org is not a
# test node with all the normal bits in place.
- name: Collect log output
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
mode: pull
src: "/var/log/ansible/{{ playbook_name }}.log"
verify_host: true
when: infra_prod_playbook_collect_log
- name: Return playbook log artifact to Zuul
when: infra_prod_playbook_collect_log
zuul_return:
data:
zuul:
artifacts:
- name: "Playbook Log"
url: "{{ playbook_name }}.log"
metadata:
type: text
# If we aren't publishing logs through zuul then keep a set on
# bridge directly.
- name: Rename playbook log on bridge
when: not infra_prod_playbook_collect_log
become: yes
copy:
remote_src: yes
src: "/var/log/ansible/{{ playbook_name }}.log"
dest: "/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}"
- name: Cleanup old playbook logs on bridge
when: not infra_prod_playbook_collect_log
become: yes
shell: |
find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete

87
playbooks/zuul/run-production-playbook.yaml
View File

@ -15,7 +15,6 @@
tasks:
- name: Run the production playbook and capture logs
block:
- name: Get a current timestamp
set_fact:
_log_timestamp: "{{ lookup('pipe', 'date +%Y-%m-%dT%H:%M:%S') }}"
@ -30,7 +29,6 @@
register: _run
always:
- name: Send run stats
shell: |
# delta is in string format h:m:s.sss; convert to ms for statsd
@ -40,88 +38,3 @@
echo 'bridge.ansible.{{ zuul.job }}.rc:{{ _run.rc }}|g' | nc -w 1 -u graphite.opendev.org 8125
args:
executable: '/bin/bash'
- name: Encrypt log
when: infra_prod_playbook_encrypt_log|default(False)
block:
- name: Create temporary staging area for encrypted logs
tempfile:
state: directory
register: _encrypt_tempdir
- name: Copy log to tempdir as Zuul user
copy:
src: '/var/log/ansible/{{ playbook_name }}.log'
dest: '{{ _encrypt_tempdir.path }}'
owner: zuul
group: zuul
mode: '0644'
remote_src: yes
become: yes
- name: Encrypt logs
include_role:
name: encrypt-logs
vars:
encrypt_logs_files:
- '{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log'
# Artifact URL should just point to root directory, so blank
encrypt_logs_artifact_path: ''
encrypt_logs_download_script_path: '{{ _encrypt_tempdir.path }}'
- name: Return logs
synchronize:
src: '{{ item[0] }}'
dest: '{{ item[1] }}'
mode: pull
verify_host: true
loop:
- ['{{ _encrypt_tempdir.path }}/{{ playbook_name }}.log.gpg', '{{ zuul.executor.log_root }}/{{ playbook_name }}.log.gpg']
- ['{{ _encrypt_tempdir.path }}/download-logs.sh' , '{{ zuul.executor.log_root }}/download-gpg-logs.sh']
always:
- name: Remove temporary staging
file:
path: '{{ _encrypt_tempdir.path }}'
state: absent
when: _encrypt_tempdir is defined
# Not using normal zuul job roles as bridge.openstack.org is not a
# test node with all the normal bits in place.
- name: Collect log output
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ playbook_name }}.log"
mode: pull
src: "/var/log/ansible/{{ playbook_name }}.log"
verify_host: true
when: infra_prod_playbook_collect_log
- name: Return playbook log artifact to Zuul
when: infra_prod_playbook_collect_log
zuul_return:
data:
zuul:
artifacts:
- name: "Playbook Log"
url: "{{ playbook_name }}.log"
metadata:
type: text
# If we aren't publishing logs through zuul then keep a set on
# bridge directly.
- name: Rename playbook log on bridge
when: not infra_prod_playbook_collect_log
become: yes
copy:
remote_src: yes
src: "/var/log/ansible/{{ playbook_name }}.log"
dest: "/var/log/ansible/{{ playbook_name }}.log.{{ _log_timestamp }}"
- name: Cleanup old playbook logs on bridge
when: not infra_prod_playbook_collect_log
become: yes
shell: |
find /var/log/ansible -name '{{ playbook_name }}.log.*' -type f -mtime +30 -delete

1
zuul.d/infra-prod.yaml
View File

@ -21,6 +21,7 @@
abstract: true
semaphores: infra-prod-playbook
run: playbooks/zuul/run-production-playbook.yaml
post-run: playbooks/zuul/run-production-playbook-post.yaml
required-projects:
- opendev/system-config
vars: