Enable Logs Output in Zuul for Deployment Scripts

This Fixes Logs for Deployment Validation.

Change-Id: I6cda8d0f0bd2d5e957abb8df9b512866f169ae59
Signed-off-by: diwakar thyagaraj <diwakar.chitoor.thyagaraj@att.com>

tools/gate/playbooks/airship-porthole-collect-logs.yaml

@@ -0,0 +1,41 @@
+# Copyright 2020 Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }} | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - gather-host-logs
+  tags:
+    - gather-host-logs
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }} | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-selenium-data
+  tags:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-selenium-data

tools/gate/playbooks/roles/describe-kubernetes-objects/tasks/main.yaml

@@ -0,0 +1,108 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating directory for cluster scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/cluster"
+    state: directory
+
+- name: "Gathering descriptions for cluster scoped objects"
+  shell: |-
+          set -e
+          export OBJECT_TYPE=node,clusterrole,clusterrolebinding,storageclass,namespace
+          export PARALLELISM_FACTOR=2
+
+          function list_objects () {
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {}
+          }
+          export -f list_objects
+
+          function name_objects () {
+            export OBJECT=$1
+            kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export OBJECT=${input[0]}
+            export NAME=${input[1]#*/}
+            echo "${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/cluster/${OBJECT}"
+            mkdir -p ${DIR}
+            kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
+            kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          list_objects | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "creating directory for namespace scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/namespaced"
+    state: directory
+
+- name: "Gathering descriptions for namespace scoped objects"
+  shell: |-
+          set -e
+          export OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets
+          export PARALLELISM_FACTOR=2
+          function get_namespaces () {
+            kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
+          }
+
+          function list_namespaced_objects () {
+            export NAMESPACE=$1
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {}
+          }
+          export -f list_namespaced_objects
+
+          function name_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            export NAME=${input[2]#*/}
+            echo "${NAMESPACE}/${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/namespaced/${NAMESPACE}/${OBJECT}"
+            mkdir -p ${DIR}
+            kubectl get -n ${NAMESPACE} ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
+            kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          get_namespaces | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/objects"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: yes

tools/gate/playbooks/roles/gather-host-logs/tasks/main.yaml

@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating directory for system status"
+  file:
+    path: "{{ logs_dir }}/system"
+    state: directory
+
+- name: "Get logs for each host"
+  become: yes
+  shell: |-
+          set -x
+          systemd-cgls --full --all --no-pager > {{ logs_dir }}/system/systemd-cgls.txt
+          ip addr > {{ logs_dir }}/system/ip-addr.txt
+          ip route > {{ logs_dir }}/system/ip-route.txt
+          lsblk > {{ logs_dir }}/system/lsblk.txt
+          mount > {{ logs_dir }}/system/mount.txt
+          docker images > {{ logs_dir }}/system/docker-images.txt
+          brctl show > {{ logs_dir }}/system/brctl-show.txt
+          ps aux --sort=-%mem > {{ logs_dir }}/system/ps.txt
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/system"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True

tools/gate/playbooks/roles/gather-pod-logs/tasks/main.yaml

@@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating directory for pod logs"
+  file:
+    path: "{{ logs_dir }}/pod-logs"
+    state: directory
+
+- name: "creating directory for failed pod logs"
+  file:
+    path: "{{ logs_dir }}/pod-logs/failed-pods"
+    state: directory
+
+- name: "retrieve all container logs, current and previous (if they exist)"
+  shell: |-
+          set -e
+          PARALLELISM_FACTOR=2
+          function get_namespaces () {
+            kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
+          }
+          function get_pods () {
+            NAMESPACE=$1
+            kubectl get pods -n ${NAMESPACE} -o name | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {}
+          }
+          export -f get_pods
+          function get_pod_logs () {
+            NAMESPACE=${1% *}
+            POD=${1#* }
+            INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.initContainers[]?.name')
+            CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.containers[].name')
+            for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
+              echo "${NAMESPACE}/${POD}/${CONTAINER}"
+              mkdir -p "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}"
+              mkdir -p "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}"
+              kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}/${CONTAINER}.txt"
+              kubectl logs --previous ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}/${CONTAINER}.txt"
+            done
+          }
+          export -f get_pod_logs
+          get_namespaces | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \
+            xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "Downloads pod logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/pod-logs"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True

tools/gate/playbooks/roles/gather-selenium-data/tasks/main.yaml

@@ -0,0 +1,31 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating directory for helm release descriptions"
+  file:
+    path: "{{ logs_dir }}/selenium"
+    state: directory
+
+- name: "Get selenium data"
+  shell: |-
+          set -x
+          cp /tmp/artifacts/* {{ logs_dir }}/selenium/.
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/selenium"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True

tools/gate/playbooks/roles/helm-release-status/tasks/main.yaml

@@ -0,0 +1,51 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating directory for helm release status"
+  file:
+    path: "{{ logs_dir }}/helm/{{ directory }}"
+    state: directory
+  loop_control:
+    loop_var: directory
+  with_items:
+    - values
+    - releases
+
+- name: "retrieve all deployed charts"
+  shell: |-
+          set -e
+          helm ls --short
+  args:
+    executable: /bin/bash
+  register: helm_releases
+  ignore_errors: True
+
+- name: "Gather get release status for helm charts"
+  shell: |-
+          set -e
+          helm status {{ helm_released }}  >> {{ logs_dir }}/helm/releases/{{ helm_release }}.txt
+          helm get values {{ helm_released }}  >> {{ logs_dir }}/helm/values/{{ helm_release }}.yaml
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+  vars:
+    helm_release: "{{ helm_released }}"
+  loop_control:
+    loop_var: helm_released
+  with_items: "{{ helm_releases.stdout_lines }}"
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/helm"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True

zuul.d/base.yaml

@@ -86,6 +86,7 @@
     description: Checks Pods for Apparmor profile
     timeout: 7200
     run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
+    post-run: tools/gate/playbooks/airship-porthole-collect-logs.yaml
     nodeset: airship-porthole-single-node
     vars:
       gate_scripts:
@@ -125,4 +126,4 @@
           Vlpomy02Nj4PkZZDKY7Eo3uf/+3aa8/hhvJursqv6VvqwYl75euT8Immo1EUw/Hw4PBpJ
           JXcwlGoK6kvXmYaG2c7yEiH2xNtTzVTai10fsalubQWR4FTmJA5LU9+HatHBa46XLy4gS
           vMHs2uJudmxhqLIhs5Er248FVjfxw7le5dPun3ir4J7Mjgypu5y8eLCMZAFIwpeqlJrVU
-          uih8ByaTlSEh6FBzle7Hb330RjSztC4pX6xiNAmtkpxhF2rEEFS66B/ixwCphQ=
+          uih8ByaTlSEh6FBzle7Hb330RjSztC4pX6xiNAmtkpxhF2rEEFS66B/ixwCphQ=