From 605faded2eb66a5a305e5af6e6e30ba6d7c67a02 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Tue, 5 Dec 2017 15:05:20 -0600 Subject: [PATCH] Gather deployment information after checks run Adds tasks to gather descriptions of kubernetes objects, logs from deployed pods, logs from helm test pods, the status of each helm release, and metrics from endpoints exposed to prometheus Change-Id: I606797c6a5d75ba446ed2c16a9710f7b0227f910 --- .zuul.yaml | 1 + .../templates/service-controller-manager.yaml | 2 +- .../templates/service-kube-metrics.yaml | 2 + .../templates/service-scheduler.yaml | 2 +- node-exporter/templates/service.yaml | 3 +- prometheus/templates/service.yaml | 2 + tools/gate/chart-deploys/default.yaml | 2 +- tools/gate/devel/start.sh | 2 +- .../deploy-helm-packages/tasks/main.yaml | 5 ++ .../tasks/util-common-helm-test.yaml | 10 ++-- .../tasks/main.yaml | 47 +++++++++++++++ .../tasks/util-common-cluster-describe.yaml | 37 ++++++++++++ .../tasks/util-common-namespace-describe.yaml | 41 +++++++++++++ .../tasks/util-namespace-describe.yaml | 34 +++++++++++ .../playbooks/gather-pod-logs/tasks/main.yaml | 38 ++++++++++++ .../tasks/util-common-gather-logs.yaml | 56 ++++++++++++++++++ .../tasks/util-container-logs.yaml | 44 ++++++++++++++ .../gather-prom-metrics/tasks/main.yaml | 57 ++++++++++++++++++ .../tasks/util-common-prom-metrics.yaml | 35 +++++++++++ .../helm-release-status/tasks/main.yaml | 38 ++++++++++++ .../tasks/util-chart-group-releases.yaml | 20 +++++++ .../tasks/util-common-release-status.yaml | 23 ++++++++ .../playbooks/osh-infra-collect-logs.yaml | 58 +++++++++++++++++++ .../playbooks/osh-infra-deploy-charts.yaml | 1 + 24 files changed, 551 insertions(+), 9 deletions(-) create mode 100644 tools/gate/playbooks/describe-kubernetes-resources/tasks/main.yaml create mode 100644 tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-cluster-describe.yaml create mode 100644 tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-namespace-describe.yaml create mode 100644 tools/gate/playbooks/describe-kubernetes-resources/tasks/util-namespace-describe.yaml create mode 100644 tools/gate/playbooks/gather-pod-logs/tasks/main.yaml create mode 100644 tools/gate/playbooks/gather-pod-logs/tasks/util-common-gather-logs.yaml create mode 100644 tools/gate/playbooks/gather-pod-logs/tasks/util-container-logs.yaml create mode 100644 tools/gate/playbooks/gather-prom-metrics/tasks/main.yaml create mode 100644 tools/gate/playbooks/gather-prom-metrics/tasks/util-common-prom-metrics.yaml create mode 100644 tools/gate/playbooks/helm-release-status/tasks/main.yaml create mode 100644 tools/gate/playbooks/helm-release-status/tasks/util-chart-group-releases.yaml create mode 100644 tools/gate/playbooks/helm-release-status/tasks/util-common-release-status.yaml create mode 100644 tools/gate/playbooks/osh-infra-collect-logs.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 0ae2ac5dd..a296caac0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -101,6 +101,7 @@ - tools/gate/playbooks/osh-infra-build.yaml - tools/gate/playbooks/osh-infra-deploy-k8s.yaml run: tools/gate/playbooks/osh-infra-deploy-charts.yaml + post-run: tools/gate/playbooks/osh-infra-collect-logs.yaml - job: name: openstack-helm-infra-ubuntu diff --git a/kube-state-metrics/templates/service-controller-manager.yaml b/kube-state-metrics/templates/service-controller-manager.yaml index 47ccd24c7..a3fcecf6a 100644 --- a/kube-state-metrics/templates/service-controller-manager.yaml +++ b/kube-state-metrics/templates/service-controller-manager.yaml @@ -23,7 +23,7 @@ kind: Service metadata: name: kube-controller-manager-discovery labels: - component: kube-controller-manager +{{ tuple $envAll "controller-manager" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: {{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} spec: diff --git a/kube-state-metrics/templates/service-kube-metrics.yaml b/kube-state-metrics/templates/service-kube-metrics.yaml index 9ea5d9168..c361413e5 100644 --- a/kube-state-metrics/templates/service-kube-metrics.yaml +++ b/kube-state-metrics/templates/service-kube-metrics.yaml @@ -22,6 +22,8 @@ apiVersion: v1 kind: Service metadata: name: {{ tuple "kube_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "kube-state-metrics" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: {{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} spec: diff --git a/kube-state-metrics/templates/service-scheduler.yaml b/kube-state-metrics/templates/service-scheduler.yaml index 7b47e9c0d..2b2e6c8d8 100644 --- a/kube-state-metrics/templates/service-scheduler.yaml +++ b/kube-state-metrics/templates/service-scheduler.yaml @@ -23,7 +23,7 @@ kind: Service metadata: name: kube-scheduler-discovery labels: - component: kube-scheduler +{{ tuple $envAll "kube-scheduler" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: {{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} spec: diff --git a/node-exporter/templates/service.yaml b/node-exporter/templates/service.yaml index 9568c544a..6eeec9012 100644 --- a/node-exporter/templates/service.yaml +++ b/node-exporter/templates/service.yaml @@ -22,7 +22,8 @@ apiVersion: v1 kind: Service metadata: name: {{ tuple "node_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - namespace: {{ .Values.endpoints.node_metrics.namespace }} + labels: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: {{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} spec: diff --git a/prometheus/templates/service.yaml b/prometheus/templates/service.yaml index b28de8f95..5caa57713 100644 --- a/prometheus/templates/service.yaml +++ b/prometheus/templates/service.yaml @@ -22,6 +22,8 @@ apiVersion: v1 kind: Service metadata: name: {{ tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} annotations: {{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} spec: diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml index 63255391f..3fb79c097 100644 --- a/tools/gate/chart-deploys/default.yaml +++ b/tools/gate/chart-deploys/default.yaml @@ -135,7 +135,7 @@ charts: timeout: 300 test: enabled: true - timeout: 300 + timeout: 600 output: false values: storage: diff --git a/tools/gate/devel/start.sh b/tools/gate/devel/start.sh index 2fc43d204..7e4261aa9 100755 --- a/tools/gate/devel/start.sh +++ b/tools/gate/devel/start.sh @@ -70,7 +70,7 @@ elif [ "x${DEPLOY}" == "xcharts" ]; then PLAYBOOKS="osh-infra-deploy-charts" elif [ "x${DEPLOY}" == "xfull" ]; then ansible_install - PLAYBOOKS="osh-infra-docker osh-infra-build osh-infra-deploy-k8s osh-infra-deploy-charts" + PLAYBOOKS="osh-infra-docker osh-infra-build osh-infra-deploy-k8s osh-infra-deploy-charts osh-infra-collect-logs" else echo "Unknown Deploy Option Selected" exit 1 diff --git a/tools/gate/playbooks/deploy-helm-packages/tasks/main.yaml b/tools/gate/playbooks/deploy-helm-packages/tasks/main.yaml index efa7c4005..779c4008e 100644 --- a/tools/gate/playbooks/deploy-helm-packages/tasks/main.yaml +++ b/tools/gate/playbooks/deploy-helm-packages/tasks/main.yaml @@ -12,6 +12,11 @@ - include: generate-dynamic-over-rides.yaml +- name: "creating directory for helm test logs" + file: + path: "{{ logs_dir }}/helm-tests" + state: directory + - name: "iterating through Helm chart groups" vars: chart_group_name: "{{ helm_chart_group.name }}" diff --git a/tools/gate/playbooks/deploy-helm-packages/tasks/util-common-helm-test.yaml b/tools/gate/playbooks/deploy-helm-packages/tasks/util-common-helm-test.yaml index b6f264da6..a926946b1 100644 --- a/tools/gate/playbooks/deploy-helm-packages/tasks/util-common-helm-test.yaml +++ b/tools/gate/playbooks/deploy-helm-packages/tasks/util-common-helm-test.yaml @@ -39,12 +39,14 @@ debug: var: test_result.stdout_lines - - name: "gathering logs for successful helm tests for {{ release }}" + - name: "gathering logs for helm tests for {{ release }}" when: - test_result | succeeded - - "'output' in test_settings" - - "test_settings.output|bool == true" - command: "kubectl logs {{ release }}-test -n {{ namespace }}" + shell: |- + set -e + kubectl logs {{ release }}-test -n {{ namespace }} >> {{ logs_dir }}/helm-tests/{{ release }}.txt + args: + executable: /bin/bash register: test_logs - name: "displaying logs for successful helm tests for {{ release }}" diff --git a/tools/gate/playbooks/describe-kubernetes-resources/tasks/main.yaml b/tools/gate/playbooks/describe-kubernetes-resources/tasks/main.yaml new file mode 100644 index 000000000..44ca9a9b0 --- /dev/null +++ b/tools/gate/playbooks/describe-kubernetes-resources/tasks/main.yaml @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Gather all deployed namespaces" + shell: |- + set -e + kubectl get namespaces -o json | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: namespaces + +- name: "Gathering descriptions for namespaced objects" + include: util-namespace-describe.yaml + vars: + namespace: "{{ namespace }}" + loop_control: + loop_var: namespace + with_items: "{{ namespaces.stdout_lines }}" + + +- name: "Gathering descriptions for cluster scoped objects" + include: util-common-cluster-describe.yaml + vars: + cluster_object: "{{ cluster_object }}" + loop_control: + loop_var: cluster_object + with_items: + - node + - clusterrole + - clusterrolebinding + - storageclass + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/resources" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: yes diff --git a/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-cluster-describe.yaml b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-cluster-describe.yaml new file mode 100644 index 000000000..536c811b0 --- /dev/null +++ b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-cluster-describe.yaml @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Kubectl describe cluster scoped objects common block + vars: + cluster_object: null + + block: + - name: "creating directory for {{ cluster_object }} descriptions" + file: path="{{ logs_dir }}/resources/{{ cluster_object }}" state=directory + + - name: "gathering names of {{ cluster_object }}s currently deployed" + shell: |- + set -e + kubectl get {{ cluster_object }} -o json | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: resource_names + + - name: "getting descriptions of {{ cluster_object }}s deployed" + shell: |- + set -e + kubectl describe {{ cluster_object }} {{ object_name }} > {{ logs_dir }}/resources/{{ cluster_object }}/{{ object_name }}.yaml + args: + executable: /bin/bash + loop_control: + loop_var: object_name + with_items: "{{ resource_names.stdout_lines }}" diff --git a/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-namespace-describe.yaml b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-namespace-describe.yaml new file mode 100644 index 000000000..94322fee0 --- /dev/null +++ b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-common-namespace-describe.yaml @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Kubectl describe namespaced object common block + vars: + api_object: null + object_namespace: null + + block: + - name: "creating directory for {{ api_object }} descriptions in {{ object_namespace }} namespace" + file: + path: "{{ logs_dir }}/resources/{{ object_namespace }}/{{ api_object }}" + state: directory + + - name: "gathering names of {{ api_object }}s currently deployed in {{ object_namespace }} namespace" + shell: |- + set -e + kubectl get {{ api_object }} --namespace={{ object_namespace }} -o json | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: namespaced_resource_names + + - name: "getting descriptions of {{ api_object }}s deployed in {{ object_namespace }} namespace" + when: namespaced_resource_names + shell: |- + set -e + kubectl describe {{ api_object }} {{ resource_name }} --namespace={{ object_namespace }} > {{ logs_dir }}/resources/{{ object_namespace }}/{{ api_object }}/{{ resource_name }}.yaml + args: + executable: /bin/bash + loop_control: + loop_var: resource_name + with_items: "{{ namespaced_resource_names.stdout_lines }}" diff --git a/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-namespace-describe.yaml b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-namespace-describe.yaml new file mode 100644 index 000000000..911dc52e3 --- /dev/null +++ b/tools/gate/playbooks/describe-kubernetes-resources/tasks/util-namespace-describe.yaml @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Kubectl describe all namespaced objects common block + vars: + api_objects: + - pod + - service + - pvc + - deployment + - statefulset + - daemonset + - serviceaccount + - endpoints + namespace: null + + block: + - name: "Describe all {{ api_object }} objects in {{ namespace }} namespace" + vars: + object_namespace: "{{ namespace }}" + api_object: "{{ api_object }}" + loop_control: + loop_var: api_object + include: util-common-namespace-describe.yaml + with_items: "{{ api_objects }}" diff --git a/tools/gate/playbooks/gather-pod-logs/tasks/main.yaml b/tools/gate/playbooks/gather-pod-logs/tasks/main.yaml new file mode 100644 index 000000000..3928be0fa --- /dev/null +++ b/tools/gate/playbooks/gather-pod-logs/tasks/main.yaml @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for pod logs" + file: + path: "{{ logs_dir }}/pod-logs" + state: directory + +- name: "retrieve all deployed namespaces" + shell: |- + set -e + kubectl get namespaces -o json | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: namespaces + +- include: util-container-logs.yaml + vars: + namespace: "{{ namespace }}" + loop_control: + loop_var: namespace + with_items: "{{ namespaces.stdout_lines }}" + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/pod-logs" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/tools/gate/playbooks/gather-pod-logs/tasks/util-common-gather-logs.yaml b/tools/gate/playbooks/gather-pod-logs/tasks/util-common-gather-logs.yaml new file mode 100644 index 000000000..aeeaca20c --- /dev/null +++ b/tools/gate/playbooks/gather-pod-logs/tasks/util-common-gather-logs.yaml @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather pod container logs common block + vars: + pod: null + + block: + - name: "Gather list of init containers in {{ pod }} pod in {{ namespace }} namespace" + shell: |- + set -e + kubectl get pod {{ pod }} -n {{ namespace }} -o json | jq -r '.spec.initContainers[].name' + args: + executable: /bin/bash + register: init_container_names + ignore_errors: True + + - name: "Gather logs from all init containers in pod {{ pod }}" + shell: |- + set -e + kubectl logs {{ pod }} -n {{ namespace }} -c {{ init_container }} >> {{ logs_dir }}/pod-logs/{{ namespace }}-{{ pod }}-{{ init_container }}.txt + args: + executable: /bin/bash + loop_control: + loop_var: init_container + with_items: "{{ init_container_names.stdout_lines }}" + ignore_errors: True + + - name: "Gather list of containers in {{ pod }} pod in {{ namespace }} namespace" + shell: |- + set -e + kubectl get pod {{ pod }} -n {{ namespace }} -o json | jq -r '.spec.containers[].name' + args: + executable: /bin/bash + register: container_names + ignore_errors: True + + - name: "Gather logs from all containers in pod {{ pod }}" + shell: |- + set -e + kubectl logs {{ pod }} -n {{ namespace }} -c {{ container }} >> {{ logs_dir }}/pod-logs/{{ namespace }}-{{ pod }}-{{ container }}.txt + args: + executable: /bin/bash + loop_control: + loop_var: container + with_items: "{{ container_names.stdout_lines }}" + ignore_errors: True diff --git a/tools/gate/playbooks/gather-pod-logs/tasks/util-container-logs.yaml b/tools/gate/playbooks/gather-pod-logs/tasks/util-container-logs.yaml new file mode 100644 index 000000000..7c1a248e2 --- /dev/null +++ b/tools/gate/playbooks/gather-pod-logs/tasks/util-container-logs.yaml @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather container logs common block + vars: + namespace: null + + block: + - name: "Gather list of pods in {{ namespace }} namespace" + shell: |- + set -e + kubectl get pods -n {{ namespace }} -o json | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: pod_names + ignore_errors: True + + - include: util-common-gather-logs.yaml + vars: + pod: "{{ pod_name }}" + loop_control: + loop_var: pod_name + with_items: "{{ pod_names.stdout_lines }}" diff --git a/tools/gate/playbooks/gather-prom-metrics/tasks/main.yaml b/tools/gate/playbooks/gather-prom-metrics/tasks/main.yaml new file mode 100644 index 000000000..90f3a8617 --- /dev/null +++ b/tools/gate/playbooks/gather-prom-metrics/tasks/main.yaml @@ -0,0 +1,57 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for helm release descriptions" + file: + path: "{{ logs_dir }}/prometheus" + state: directory + +- name: "get exporter services in kube-system namespace" + shell: |- + set -e + kubectl get svc -l component=metrics -n kube-system -o json \ + | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: kube_system_exporters + +- include: util-common-prom-metrics.yaml + vars: + exporter: "{{ kube_system_exporter }}" + namespace: kube-system + loop_control: + loop_var: kube_system_exporter + with_items: "{{ kube_system_exporters.stdout_lines }}" + +- name: "get exporter services in openstack namespace" + shell: |- + set -e + kubectl get svc -l component=metrics -n openstack -o json \ + | jq -r '.items[].metadata.name' + args: + executable: /bin/bash + register: openstack_exporters + +- include: util-common-prom-metrics.yaml + vars: + exporter: "{{ openstack_exporter }}" + namespace: openstack + loop_control: + loop_var: openstack_exporter + with_items: "{{ openstack_exporters.stdout_lines }}" + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/prometheus" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True diff --git a/tools/gate/playbooks/gather-prom-metrics/tasks/util-common-prom-metrics.yaml b/tools/gate/playbooks/gather-prom-metrics/tasks/util-common-prom-metrics.yaml new file mode 100644 index 000000000..0fb4b50aa --- /dev/null +++ b/tools/gate/playbooks/gather-prom-metrics/tasks/util-common-prom-metrics.yaml @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather prometheus exporter metrics common block + vars: + exporter: null + namespace: null + + block: + - name: "Get {{ exporter }} exporter service port" + shell: |- + set -e + kubectl get svc "{{ exporter }}" -n "{{ namespace }}" -o json \ + | jq -r '.spec.ports[].port' + args: + executable: /bin/bash + register: exporter_port + ignore_errors: True + + - name: "Gather metrics from {{ exporter }} exporter metrics port" + shell: |- + set -e + curl "{{ exporter }}"."{{ namespace }}":"{{ exporter_port.stdout }}"/metrics >> "{{ logs_dir }}"/prometheus/"{{ exporter }}".txt + args: + executable: /bin/bash + ignore_errors: True diff --git a/tools/gate/playbooks/helm-release-status/tasks/main.yaml b/tools/gate/playbooks/helm-release-status/tasks/main.yaml new file mode 100644 index 000000000..00fa514c9 --- /dev/null +++ b/tools/gate/playbooks/helm-release-status/tasks/main.yaml @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "creating directory for helm release status" + file: + path: "{{ logs_dir }}/helm-releases" + state: directory + +- name: "Gathering release status in chart groups" + vars: + chart_group_name: "{{ helm_chart_group.name }}" + include: util-chart-group-releases.yaml + loop_control: + loop_var: helm_chart_group + with_items: "{{ chart_groups }}" + +- name: "Downloads helm release statuses to executor" + synchronize: + src: "{{ logs_dir }}/helm-releases" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: yes + +- name: "Download helm release test logs to executor" + synchronize: + src: "{{ logs_dir }}/helm-tests" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: yes diff --git a/tools/gate/playbooks/helm-release-status/tasks/util-chart-group-releases.yaml b/tools/gate/playbooks/helm-release-status/tasks/util-chart-group-releases.yaml new file mode 100644 index 000000000..7fddb3180 --- /dev/null +++ b/tools/gate/playbooks/helm-release-status/tasks/util-chart-group-releases.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: "Gathering release status for {{ helm_chart_group.name }} chart group" + vars: + release: "{{ charts[helm_chart].release }}" + namespace: "{{ charts[helm_chart].namespace }}" + loop_control: + loop_var: helm_chart + include: util-common-release-status.yaml + with_items: "{{ helm_chart_group.charts }}" diff --git a/tools/gate/playbooks/helm-release-status/tasks/util-common-release-status.yaml b/tools/gate/playbooks/helm-release-status/tasks/util-common-release-status.yaml new file mode 100644 index 000000000..42fade4b7 --- /dev/null +++ b/tools/gate/playbooks/helm-release-status/tasks/util-common-release-status.yaml @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Helm release status common block + vars: + release: null + namespace: null + block: + - name: "describing the {{ release }} release" + shell: |- + helm status "{{ release }}" > "{{ logs_dir }}"/helm-releases/"{{ release }}".yaml + args: + executable: /bin/bash + ignore_errors: True diff --git a/tools/gate/playbooks/osh-infra-collect-logs.yaml b/tools/gate/playbooks/osh-infra-collect-logs.yaml new file mode 100644 index 000000000..0744ae3fc --- /dev/null +++ b/tools/gate/playbooks/osh-infra-collect-logs.yaml @@ -0,0 +1,58 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + vars_files: + - vars.yaml + - ../chart-deploys/default.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - helm-release-status + tags: + - helm-release-status + +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - describe-kubernetes-resources + tags: + - describe-kubernetes-resources + +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - gather-pod-logs + tags: + - gather-pod-logs + +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - gather-prom-metrics + tags: + - gather-prom-metrics diff --git a/tools/gate/playbooks/osh-infra-deploy-charts.yaml b/tools/gate/playbooks/osh-infra-deploy-charts.yaml index bc66ca0b7..b991e9e46 100644 --- a/tools/gate/playbooks/osh-infra-deploy-charts.yaml +++ b/tools/gate/playbooks/osh-infra-deploy-charts.yaml @@ -29,6 +29,7 @@ - ../chart-deploys/default.yaml vars: work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" roles: - deploy-helm-packages tags: