From 8cbd40cbc5de6eb8be45c096a4acb91e9253d410 Mon Sep 17 00:00:00 2001 From: Vamsi Savaram Date: Wed, 18 Mar 2020 21:03:27 +0100 Subject: [PATCH] Gather container logs in check pipeline This PS introduces two new roles: - airship-images-configure-docker: Configures docker logging driver to journald - airship-gather-runtime-logs Collects all journald logs for docker and containerd Also changes have been made to debian-isogen Makefile to completely output docker build logs instead of just image ID. Relates-To: #89 Change-Id: I435106b2ad15921367174446707179f157df7946 Signed-off-by: Vamsi Savaram --- .zuul.yaml | 2 + debian-isogen/Makefile | 9 ++-- playbooks/airship-collect-logs.yaml | 19 ++++++++ playbooks/airship-images-deploy-docker.yaml | 1 + .../tasks/main.yaml | 45 +++++++++++++++++++ .../tasks/main.yaml | 43 ++++++++++++++++++ .../vars/main.yaml | 20 +++++++++ 7 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 playbooks/airship-collect-logs.yaml create mode 100644 roles/airship-gather-runtime-logs/tasks/main.yaml create mode 100644 roles/airship-images-configure-docker/tasks/main.yaml create mode 100644 roles/airship-images-configure-docker/vars/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index eb9f901..3fa8445 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -28,6 +28,7 @@ name: airship-images-functional pre-run: playbooks/airship-images-deploy-docker.yaml run: playbooks/airship-images-test.yaml + post-run: playbooks/airship-collect-logs.yaml nodeset: airship-images-single-node - job: @@ -46,6 +47,7 @@ nodeset: airship-images-single-node pre-run: playbooks/airship-images-deploy-docker.yaml run: playbooks/airship-images-build.yaml + post-run: playbooks/airship-collect-logs.yaml - job: name: airship-images-publish diff --git a/debian-isogen/Makefile b/debian-isogen/Makefile index a19e49a..d332a20 100644 --- a/debian-isogen/Makefile +++ b/debian-isogen/Makefile @@ -79,7 +79,7 @@ build_isogen: mkdir -p $(BUILD_DIR) ifeq ($(IMAGE_ID), none) ifeq ($(USE_PROXY), true) - docker build . --quiet \ + docker build . \ --tag $(IMAGE) \ --label $(LABEL) \ --label "org.opencontainers.image.revision=$(COMMIT)" \ @@ -92,17 +92,18 @@ ifeq ($(USE_PROXY), true) --build-arg HTTPS_PROXY=$(PROXY) \ --build-arg no_proxy=$(NO_PROXY) \ --build-arg NO_PROXY=$(NO_PROXY) \ - --build-arg GIT_COMMIT=$(COMMIT) > $(BUILD_DIR)/image_id + --build-arg GIT_COMMIT=$(COMMIT) else - docker build . --quiet \ + docker build . \ --tag $(IMAGE) \ --label $(LABEL) \ --label "org.opencontainers.image.revision=$(COMMIT)" \ --label "org.opencontainers.image.created=\ $(shell date --rfc-3339=seconds --utc)" \ --label "org.opencontainers.image.title=$(IMAGE_NAME)" \ - --build-arg GIT_COMMIT=$(COMMIT) > $(BUILD_DIR)/image_id + --build-arg GIT_COMMIT=$(COMMIT) endif + echo $(shell docker images -q $(IMAGE)) > $(BUILD_DIR)/image_id else echo $(IMAGE_ID) > $(BUILD_DIR)/image_id endif diff --git a/playbooks/airship-collect-logs.yaml b/playbooks/airship-collect-logs.yaml new file mode 100644 index 0000000..91167ad --- /dev/null +++ b/playbooks/airship-collect-logs.yaml @@ -0,0 +1,19 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + vars: + logs_dir: "/tmp/logs" + roles: + - airship-gather-runtime-logs diff --git a/playbooks/airship-images-deploy-docker.yaml b/playbooks/airship-images-deploy-docker.yaml index 83f61d2..abfc5da 100644 --- a/playbooks/airship-images-deploy-docker.yaml +++ b/playbooks/airship-images-deploy-docker.yaml @@ -15,3 +15,4 @@ - hosts: all roles: - install-docker + - airship-images-configure-docker diff --git a/roles/airship-gather-runtime-logs/tasks/main.yaml b/roles/airship-gather-runtime-logs/tasks/main.yaml new file mode 100644 index 0000000..84a3f12 --- /dev/null +++ b/roles/airship-gather-runtime-logs/tasks/main.yaml @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: populate service facts + service_facts: + +- name: set runtime logs dir + set_fact: + runtime_logs_dir: "{{ logs_dir }}/runtime" + +- name: ensure directory for runtime logs exists + file: + state: directory + path: "{{ runtime_logs_dir }}" + +- name: dump docker logs + shell: |- + journalctl --unit "docker" --no-pager > "{{ runtime_logs_dir }}/docker.log" + when: ansible_facts.services['docker.service'] is defined + args: + executable: /bin/bash + become: true + +- name: dump containerd logs + shell: |- + journalctl --unit "containerd" --no-pager > "{{ runtime_logs_dir }}/containerd.log" + when: ansible_facts.services['containerd.service'] is defined + args: + executable: /bin/bash + become: true + +- name: "Downloads logs to executor" + synchronize: + src: "{{ runtime_logs_dir }}" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull diff --git a/roles/airship-images-configure-docker/tasks/main.yaml b/roles/airship-images-configure-docker/tasks/main.yaml new file mode 100644 index 0000000..c836672 --- /dev/null +++ b/roles/airship-images-configure-docker/tasks/main.yaml @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check if docker daemon configuration exists + stat: + path: "{{ docker_config_path }}/daemon.json" + register: docker_config_stat + +- name: Load docker daemon configuration + slurp: + path: "{{ docker_config_path }}/daemon.json" + register: docker_config + when: docker_config_stat.stat.exists + +- name: Parse docker daemon configuration + set_fact: + docker_config: "{{ docker_config.content | b64decode | from_json }}" + when: docker_config_stat.stat.exists + +- name: Append to docker daemon configuration + set_fact: + docker_config: "{{ docker_config | default({}) | combine(docker_config_append) }}" + +- name: Save docker daemon configuration + copy: + content: "{{ docker_config | to_nice_json }}" + dest: "{{ docker_config_path }}/daemon.json" + become: true + +- name: "Restart docker service" + service: + name: docker + state: restarted + become: true diff --git a/roles/airship-images-configure-docker/vars/main.yaml b/roles/airship-images-configure-docker/vars/main.yaml new file mode 100644 index 0000000..5a94f00 --- /dev/null +++ b/roles/airship-images-configure-docker/vars/main.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker_config_path: "/etc/docker" + +docker_config_log_driver: "journald" +docker_config_log_opts: {} + +docker_config_append: + "log-driver": "{{ docker_config_log_driver }}" + "log-opts": "{{ docker_config_log_opts }}"