Vino collect logs of Kubernetes objects

Change-Id: I6224987aacc72f339f631ba9c85d9fc22d4bab4f

playbooks/vino-collect-logs.yaml

@@ -19,3 +19,4 @@
     LIBVIRT_DEFAULT_URI: qemu:///system
   roles:
     - vino-gather-pod-logs
+    - vino-describe-kubernetes-objects

roles/vino-describe-kubernetes-objects/defaults/main.yaml

@@ -0,0 +1,15 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+kubeconfig: "{{ airshipctl_config_dir_default | default(ansible_env.HOME) }}/.kube/config"
+...

roles/vino-describe-kubernetes-objects/tasks/get-contexts.yaml

@@ -0,0 +1,22 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "Save kubeconfig contexts"
+  shell: |-
+    kubectl --kubeconfig {{ kubeconfig }} config get-contexts -o name
+  register: kctl_context_command
+
+- name: "Save kubeconfig contexts to list"
+  set_fact:
+    kctl_context_list: "{{ kctl_context_command.stdout.split('\n') }}"
+...

roles/vino-describe-kubernetes-objects/tasks/main.yaml

@@ -0,0 +1,67 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## This task will populate kctl_context_list variable with list of contexts if they exist
+---
+- name: "Set cluster scoped collectibles"
+  set_fact:
+    collect_clustered_objects:
+      - node
+      - clusterrole
+      - clusterrolebinding
+      - storageclass
+      - namespace
+      - persistentvolume
+      - customresourcedefinition
+
+- name: "Set namespace scoped collectibles"
+  set_fact:
+    collect_namespaced_objects:
+      - baremetalhosts
+      - configmaps
+      - cronjobs
+      - daemonsets
+      - deployments
+      - endpoints
+      - ingresses
+      - jobs
+      - networkpolicies
+      - pods
+      - podsecuritypolicies
+      - persistentvolumeclaims
+      - replicasets
+      - rolebindings
+      - roles
+      - secrets
+      - serviceaccounts
+      - services
+      - statefulsets
+      - vino
+
+
+- name: "Get context list"
+  include_tasks: get-contexts.yaml
+
+- name: "Save logs for each context"
+  include_tasks: save-context-objects.yaml
+  when: kctl_context_list | length > 0
+  with_items: "{{ kctl_context_list }}"
+  loop_control:
+    loop_var: kctl_context
+
+- name: "Downloads logs to executor"
+  synchronize:
+    src: "{{ logs_dir }}/objects"
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+    mode: pull
+  ignore_errors: True
+...

roles/vino-describe-kubernetes-objects/tasks/save-context-objects.yaml

@@ -0,0 +1,110 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "creating directory for cluster scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/{{ kctl_context }}/cluster"
+    state: directory
+
+- name: "Gathering descriptions for cluster scoped objects"
+  shell: |-
+          set -e
+          export KUBECONFIG="{{ kubeconfig }}"
+          export CONTEXT={{ kctl_context }}
+          export OBJECT_TYPE="{{ collect_clustered_objects | join(',') }}"
+          export PARALLELISM_FACTOR=2
+
+          function list_objects () {
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "$@"' _ {}
+          }
+          export -f list_objects
+
+          function name_objects () {
+            export OBJECT=$1
+            kubectl get ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${OBJECT} ${1#*/}"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export OBJECT=${input[0]}
+            export NAME=${input[1]#*/}
+            echo "${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/${CONTEXT}/cluster/${OBJECT}"
+            mkdir -p ${DIR}
+            kubectl get ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml"
+            kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          list_objects | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+
+- name: "creating directory for namespace scoped objects"
+  file:
+    path: "{{ logs_dir }}/objects/{{ kctl_context }}/namespaced"
+    state: directory
+
+- name: "Gathering descriptions for namespace scoped objects"
+  shell: |-
+          set -e
+          export KUBECONFIG="{{ kubeconfig }}"
+          export CONTEXT={{ kctl_context }}
+          export OBJECT_TYPE="{{ collect_namespaced_objects | join(',') }}"
+          export TMP_DIR=$(mktemp -d)
+          export PARALLELISM_FACTOR=2
+          printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P ${PARALLELISM_FACTOR} -n1 bash -c 'kubectl get -A -oyaml "$@" > ${TMP_DIR}/"$@".yaml' _ {}
+
+          function get_namespaces () {
+            kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
+          }
+
+          function list_namespaced_objects () {
+            export NAMESPACE=$1
+            printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {}
+          }
+          export -f list_namespaced_objects
+
+          function name_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {}
+          }
+          export -f name_objects
+
+          function get_objects () {
+            input=($1)
+            export NAMESPACE=${input[0]}
+            export OBJECT=${input[1]}
+            export NAME=${input[2]#*/}
+            echo "${NAMESPACE}/${OBJECT}/${NAME}"
+            DIR="{{ logs_dir }}/objects/${CONTEXT}/namespaced/${NAMESPACE}/${OBJECT}"
+            mkdir -p ${DIR}
+            cat "${TMP_DIR}/${OBJECT}.yaml" | kustomize cfg grep "metadata.namespace=${NAMESPACE}" | kustomize cfg grep "metadata.name=${NAME}" > "${DIR}/${NAME}.yaml"
+            kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt"
+          }
+          export -f get_objects
+
+          get_namespaces | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \
+            xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {}
+  args:
+    executable: /bin/bash
+  ignore_errors: True
+...

roles/vino-gather-pod-logs/tasks/save-context-logs.yaml

@@ -49,7 +49,6 @@
             done
           }
           export -f get_pod_logs
-          kubectl config use-context {{ kctl_context }}
           get_namespaces | \
             xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \
             xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {}