From 4a2d6deefa7236134d7ca3942daea0694f42dc05 Mon Sep 17 00:00:00 2001 From: diwakar thyagaraj Date: Wed, 4 Mar 2020 17:54:11 +0000 Subject: [PATCH] Enable Logs Output in Zuul for Deployment Scripts This Fixes Logs for Deployment Validation. Change-Id: I6cda8d0f0bd2d5e957abb8df9b512866f169ae59 Signed-off-by: diwakar thyagaraj --- .../airship-porthole-collect-logs.yaml | 41 +++++++ .../tasks/main.yaml | 108 ++++++++++++++++++ .../roles/gather-host-logs/tasks/main.yaml | 39 +++++++ .../roles/gather-pod-logs/tasks/main.yaml | 61 ++++++++++ .../gather-selenium-data/tasks/main.yaml | 31 +++++ .../roles/helm-release-status/tasks/main.yaml | 51 +++++++++ zuul.d/base.yaml | 3 +- 7 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 tools/gate/playbooks/airship-porthole-collect-logs.yaml create mode 100644 tools/gate/playbooks/roles/describe-kubernetes-objects/tasks/main.yaml create mode 100644 tools/gate/playbooks/roles/gather-host-logs/tasks/main.yaml create mode 100644 tools/gate/playbooks/roles/gather-pod-logs/tasks/main.yaml create mode 100644 tools/gate/playbooks/roles/gather-selenium-data/tasks/main.yaml create mode 100644 tools/gate/playbooks/roles/helm-release-status/tasks/main.yaml diff --git a/tools/gate/playbooks/airship-porthole-collect-logs.yaml b/tools/gate/playbooks/airship-porthole-collect-logs.yaml new file mode 100644 index 00000000..01c9ad90 --- /dev/null +++ b/tools/gate/playbooks/airship-porthole-collect-logs.yaml @@ -0,0 +1,41 @@ +# Copyright 2020 Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }} | default('') }}" + logs_dir: "/tmp/logs" + roles: + - gather-host-logs + tags: + - gather-host-logs + +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }} | default('') }}" + logs_dir: "/tmp/logs" + roles: + - helm-release-status + - describe-kubernetes-objects + - gather-pod-logs + - gather-selenium-data + tags: + - helm-release-status + - describe-kubernetes-objects + - gather-pod-logs + - gather-selenium-data diff --git a/tools/gate/playbooks/roles/describe-kubernetes-objects/tasks/main.yaml b/tools/gate/playbooks/roles/describe-kubernetes-objects/tasks/main.yaml new file mode 100644 index 00000000..bbd2bad3 --- /dev/null +++ b/tools/gate/playbooks/roles/describe-kubernetes-objects/tasks/main.yaml @@ -0,0 +1,108 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for cluster scoped objects" + file: + path: "{{ logs_dir }}/objects/cluster" + state: directory + +- name: "Gathering descriptions for cluster scoped objects" + shell: |- + set -e + export OBJECT_TYPE=node,clusterrole,clusterrolebinding,storageclass,namespace + export PARALLELISM_FACTOR=2 + + function list_objects () { + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {} + } + export -f list_objects + + function name_objects () { + export OBJECT=$1 + kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export OBJECT=${input[0]} + export NAME=${input[1]#*/} + echo "${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/cluster/${OBJECT}" + mkdir -p ${DIR} + kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml" + kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + list_objects | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "creating directory for namespace scoped objects" + file: + path: "{{ logs_dir }}/objects/namespaced" + state: directory + +- name: "Gathering descriptions for namespace scoped objects" + shell: |- + set -e + export OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets + export PARALLELISM_FACTOR=2 + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + + function list_namespaced_objects () { + export NAMESPACE=$1 + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {} + } + export -f list_namespaced_objects + + function name_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + export NAME=${input[2]#*/} + echo "${NAMESPACE}/${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/namespaced/${NAMESPACE}/${OBJECT}" + mkdir -p ${DIR} + kubectl get -n ${NAMESPACE} ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml" + kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/objects" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: yes diff --git a/tools/gate/playbooks/roles/gather-host-logs/tasks/main.yaml b/tools/gate/playbooks/roles/gather-host-logs/tasks/main.yaml new file mode 100644 index 00000000..29f028e3 --- /dev/null +++ b/tools/gate/playbooks/roles/gather-host-logs/tasks/main.yaml @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for system status" + file: + path: "{{ logs_dir }}/system" + state: directory + +- name: "Get logs for each host" + become: yes + shell: |- + set -x + systemd-cgls --full --all --no-pager > {{ logs_dir }}/system/systemd-cgls.txt + ip addr > {{ logs_dir }}/system/ip-addr.txt + ip route > {{ logs_dir }}/system/ip-route.txt + lsblk > {{ logs_dir }}/system/lsblk.txt + mount > {{ logs_dir }}/system/mount.txt + docker images > {{ logs_dir }}/system/docker-images.txt + brctl show > {{ logs_dir }}/system/brctl-show.txt + ps aux --sort=-%mem > {{ logs_dir }}/system/ps.txt + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/system" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/tools/gate/playbooks/roles/gather-pod-logs/tasks/main.yaml b/tools/gate/playbooks/roles/gather-pod-logs/tasks/main.yaml new file mode 100644 index 00000000..8f48b7da --- /dev/null +++ b/tools/gate/playbooks/roles/gather-pod-logs/tasks/main.yaml @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for pod logs" + file: + path: "{{ logs_dir }}/pod-logs" + state: directory + +- name: "creating directory for failed pod logs" + file: + path: "{{ logs_dir }}/pod-logs/failed-pods" + state: directory + +- name: "retrieve all container logs, current and previous (if they exist)" + shell: |- + set -e + PARALLELISM_FACTOR=2 + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + function get_pods () { + NAMESPACE=$1 + kubectl get pods -n ${NAMESPACE} -o name | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {} + } + export -f get_pods + function get_pod_logs () { + NAMESPACE=${1% *} + POD=${1#* } + INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.initContainers[]?.name') + CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.containers[].name') + for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do + echo "${NAMESPACE}/${POD}/${CONTAINER}" + mkdir -p "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}" + mkdir -p "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}" + kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}/${CONTAINER}.txt" + kubectl logs --previous ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}/${CONTAINER}.txt" + done + } + export -f get_pod_logs + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \ + xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads pod logs to executor" + synchronize: + src: "{{ logs_dir }}/pod-logs" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/tools/gate/playbooks/roles/gather-selenium-data/tasks/main.yaml b/tools/gate/playbooks/roles/gather-selenium-data/tasks/main.yaml new file mode 100644 index 00000000..3fcc9ca7 --- /dev/null +++ b/tools/gate/playbooks/roles/gather-selenium-data/tasks/main.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for helm release descriptions" + file: + path: "{{ logs_dir }}/selenium" + state: directory + +- name: "Get selenium data" + shell: |- + set -x + cp /tmp/artifacts/* {{ logs_dir }}/selenium/. + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/selenium" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/tools/gate/playbooks/roles/helm-release-status/tasks/main.yaml b/tools/gate/playbooks/roles/helm-release-status/tasks/main.yaml new file mode 100644 index 00000000..b73250af --- /dev/null +++ b/tools/gate/playbooks/roles/helm-release-status/tasks/main.yaml @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for helm release status" + file: + path: "{{ logs_dir }}/helm/{{ directory }}" + state: directory + loop_control: + loop_var: directory + with_items: + - values + - releases + +- name: "retrieve all deployed charts" + shell: |- + set -e + helm ls --short + args: + executable: /bin/bash + register: helm_releases + ignore_errors: True + +- name: "Gather get release status for helm charts" + shell: |- + set -e + helm status {{ helm_released }} >> {{ logs_dir }}/helm/releases/{{ helm_release }}.txt + helm get values {{ helm_released }} >> {{ logs_dir }}/helm/values/{{ helm_release }}.yaml + args: + executable: /bin/bash + ignore_errors: True + vars: + helm_release: "{{ helm_released }}" + loop_control: + loop_var: helm_released + with_items: "{{ helm_releases.stdout_lines }}" + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/helm" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 34189634..4f5a8226 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -86,6 +86,7 @@ description: Checks Pods for Apparmor profile timeout: 7200 run: tools/gate/playbooks/airship-porthole-gate-runner.yaml + post-run: tools/gate/playbooks/airship-porthole-collect-logs.yaml nodeset: airship-porthole-single-node vars: gate_scripts: @@ -125,4 +126,4 @@ Vlpomy02Nj4PkZZDKY7Eo3uf/+3aa8/hhvJursqv6VvqwYl75euT8Immo1EUw/Hw4PBpJ JXcwlGoK6kvXmYaG2c7yEiH2xNtTzVTai10fsalubQWR4FTmJA5LU9+HatHBa46XLy4gS vMHs2uJudmxhqLIhs5Er248FVjfxw7le5dPun3ir4J7Mjgypu5y8eLCMZAFIwpeqlJrVU - uih8ByaTlSEh6FBzle7Hb330RjSztC4pX6xiNAmtkpxhF2rEEFS66B/ixwCphQ= \ No newline at end of file + uih8ByaTlSEh6FBzle7Hb330RjSztC4pX6xiNAmtkpxhF2rEEFS66B/ixwCphQ=