From fc52cda24a511f47b5136feb3dbacf0fb6eb3f61 Mon Sep 17 00:00:00 2001 From: rakesh-patnaik Date: Mon, 11 Dec 2017 07:26:10 +0000 Subject: [PATCH] Openstack metrics exporter for prometheus This exporter provides a means for Prometheus to gather openstack service metrics related to overlying openstack-helm deployments Change-Id: I5f1789c62b4547add0c67edb51540f712bf43da8 --- prometheus-openstack-exporter/Chart.yaml | 24 +++ .../requirements.yaml | 19 ++ .../templates/bin/_openstack-exporter.sh.tpl | 30 ++++ .../templates/configmap-bin.yaml | 29 +++ .../templates/deployment.yaml | 79 ++++++++ .../templates/job-image-repo-sync.yaml | 69 +++++++ .../secret-openstack-metrics-user.yaml | 29 +++ .../templates/service.yaml | 36 ++++ prometheus-openstack-exporter/values.yaml | 168 ++++++++++++++++++ tools/gate/chart-deploys/default.yaml | 15 ++ 10 files changed, 498 insertions(+) create mode 100644 prometheus-openstack-exporter/Chart.yaml create mode 100644 prometheus-openstack-exporter/requirements.yaml create mode 100644 prometheus-openstack-exporter/templates/bin/_openstack-exporter.sh.tpl create mode 100644 prometheus-openstack-exporter/templates/configmap-bin.yaml create mode 100644 prometheus-openstack-exporter/templates/deployment.yaml create mode 100644 prometheus-openstack-exporter/templates/job-image-repo-sync.yaml create mode 100644 prometheus-openstack-exporter/templates/secret-openstack-metrics-user.yaml create mode 100644 prometheus-openstack-exporter/templates/service.yaml create mode 100644 prometheus-openstack-exporter/values.yaml diff --git a/prometheus-openstack-exporter/Chart.yaml b/prometheus-openstack-exporter/Chart.yaml new file mode 100644 index 000000000..ef292c19d --- /dev/null +++ b/prometheus-openstack-exporter/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack Metrics Exporter for Prometheus +name: prometheus-openstack-exporter +version: 0.1.0 +home: https://github.com/openstack/openstack-helm-infra +sources: + - https://git.openstack.org/cgit/openstack/openstack-helm-infra + - https://github.com/rakesh-patnaik/prometheus-openstack-exporter +maintainers: + - name: OpenStack-Helm Authors diff --git a/prometheus-openstack-exporter/requirements.yaml b/prometheus-openstack-exporter/requirements.yaml new file mode 100644 index 000000000..00a045b4e --- /dev/null +++ b/prometheus-openstack-exporter/requirements.yaml @@ -0,0 +1,19 @@ + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts + version: 0.1.0 diff --git a/prometheus-openstack-exporter/templates/bin/_openstack-exporter.sh.tpl b/prometheus-openstack-exporter/templates/bin/_openstack-exporter.sh.tpl new file mode 100644 index 000000000..66f1cb40f --- /dev/null +++ b/prometheus-openstack-exporter/templates/bin/_openstack-exporter.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec python /usr/local/bin/prometheus_openstack_exporter/exporter.py +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/prometheus-openstack-exporter/templates/configmap-bin.yaml b/prometheus-openstack-exporter/templates/configmap-bin.yaml new file mode 100644 index 000000000..7674d116c --- /dev/null +++ b/prometheus-openstack-exporter/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openstack-exporter-bin +data: + image-repo-sync.sh: |+ +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + openstack-exporter.sh: | +{{ tuple "bin/_openstack-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/deployment.yaml b/prometheus-openstack-exporter/templates/deployment.yaml new file mode 100644 index 000000000..b90ed0d39 --- /dev/null +++ b/prometheus-openstack-exporter/templates/deployment.yaml @@ -0,0 +1,79 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +{{- $ksUserSecret := .Values.secrets.identity.user }} +{{- if .Values.images.local_registry.active -}} +{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.openstack_metrics_exporter .Values.conditional_dependencies.local_image_registry) -}} +{{- else -}} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.openstack_metrics_exporter -}} +{{- end -}} + +{{- $serviceAccountName := "prometheus-openstack-exporter" }} +{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: openstack-exporter +spec: + replicas: {{ .Values.pod.replicas.openstack_metrics_exporter }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.openstack_metrics_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: openstack-metrics-exporter +{{ tuple $envAll "openstack_metrics_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.openstack_metrics_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/openstack-exporter.sh + - start + ports: + - name: metrics + containerPort: {{ .Values.network.openstack_metrics_exporter.port }} + env: + - name: TIMEOUT_SECONDS + value: "{{ .Values.conf.openstack_metrics_exporter.timeout_seconds }}" + - name: OS_POLLING_INTERVAL + value: "{{ .Values.conf.openstack_metrics_exporter.polling_interval_seconds }}" + - name: OS_RETRIES + value: "{{ .Values.conf.openstack_metrics_exporter.retries }}" + - name: LISTEN_PORT + value: "{{ .Values.network.openstack_metrics_exporter.port }}" +{{- with $env := dict "ksUserSecret" $ksUserSecret }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + volumeMounts: + - name: openstack-exporter-bin + mountPath: /tmp/openstack-exporter.sh + subPath: openstack-exporter.sh + readOnly: true + volumes: + - name: openstack-exporter-bin + configMap: + name: openstack-exporter-bin + defaultMode: 0555 +{{- end }} diff --git a/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml b/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..5cdc4185b --- /dev/null +++ b/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml @@ -0,0 +1,69 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_image_repo_sync }} +{{- $envAll := . }} +{{- if .Values.images.local_registry.active -}} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.image_repo_sync -}} + +{{- $serviceAccountName := "openstack-exporter-image-repo-sync"}} +{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: openstack-metrics-exporter-image-repo-sync +spec: + template: + metadata: + labels: +{{ tuple $envAll "openstack-metrics-exporter" "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: image-repo-sync +{{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.image_repo_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: LOCAL_REPO + value: "{{ tuple "local_image_registry" "node" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "local_image_registry" "node" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: IMAGE_SYNC_LIST + value: "{{ include "helm-toolkit.utils.image_sync_list" . }}" + command: + - /tmp/image-repo-sync.sh + volumeMounts: + - name: openstack-exporter-bin + mountPath: /tmp/image-repo-sync.sh + subPath: image-repo-sync.sh + readOnly: true + - name: docker-socket + mountPath: /var/run/docker.sock + volumes: + - name: openstack-exporter-bin + configMap: + name: openstack-exporter-bin + defaultMode: 0555 + - name: docker-socket + hostPath: + path: /var/run/docker.sock +{{- end }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/secret-openstack-metrics-user.yaml b/prometheus-openstack-exporter/templates/secret-openstack-metrics-user.yaml new file mode 100644 index 000000000..1e6deb3da --- /dev/null +++ b/prometheus-openstack-exporter/templates/secret-openstack-metrics-user.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.openstack_metrics_user }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.identity.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ $envAll.Values.endpoints.openstack_metrics_exporter.namespace }} +type: Opaque +data: +{{- tuple "user" "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/service.yaml b/prometheus-openstack-exporter/templates/service.yaml new file mode 100644 index 000000000..e498d13bf --- /dev/null +++ b/prometheus-openstack-exporter/templates/service.yaml @@ -0,0 +1,36 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_openstack_metrics_exporter }} +{{- $envAll := . }} +{{- $endpoint := $envAll.Values.endpoints.openstack_metrics_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "openstack_metrics_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "openstack-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{ tuple $endpoint $envAll | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +spec: + ports: + - name: http + port: {{ .Values.network.openstack_metrics_exporter.port }} + targetPort: {{ .Values.network.openstack_metrics_exporter.port }} + selector: +{{ tuple $envAll "openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-openstack-exporter/values.yaml b/prometheus-openstack-exporter/values.yaml new file mode 100644 index 000000000..1775a2075 --- /dev/null +++ b/prometheus-openstack-exporter/values.yaml @@ -0,0 +1,168 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for kube-state-metrics. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +images: + tags: + openstack_metrics_exporter: docker.io/rakeshpatnaik/prometheus-openstack-exporter:v0.1 + helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3 + pull_policy: IfNotPresent + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + mounts: + openstack_metrics_exporter: + openstack_metrics_exporter: + init_container: null + replicas: + openstack_metrics_exporter: 1 + lifecycle: + upgrades: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + openstack_metrics_exporter: + timeout: 30 + resources: + enabled: false + kube_state_metrics: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +secrets: + identity: + user: openstack-metrics-user + +dependencies: + image_repo_sync: + services: + - service: local_image_registry + endpoint: internal + +conditional_dependencies: + local_image_registry: + jobs: + - openstack-metrics-exporter-image-repo-sync + services: + - service: local_image_registry + endpoint: node + +conf: + openstack_metrics_exporter: + polling_interval_seconds: 30 + timeout_seconds: 20 + retries: 1 + os_cpu_oc_ratio: 1.5 + os_ram_oc_ratio: 1.0 + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + openstack_metrics_exporter: + namespace: null + hosts: + default: openstack-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + scrape: true + identity: + name: keystone + auth: + user: + role: admin + region_name: RegionOne + username: nova + password: password + project_name: service + user_domain_name: default + project_domain_name: default + hosts: + default: keystone-api + public: keystone + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + admin: + default: 35357 + api: + default: 80 + +network: + openstack_metrics_exporter: + port: 9103 + +manifests: + configmap_bin: true + clusterrole: true + clusterrolebinding: true + deployment: true + job_image_repo_sync: true + rbac_entrypoint: true + service_openstack_metrics_exporter: true + serviceaccount: true + openstack_metrics_user: true diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml index 911eaccb8..750a19aab 100644 --- a/tools/gate/chart-deploys/default.yaml +++ b/tools/gate/chart-deploys/default.yaml @@ -29,6 +29,11 @@ chart_groups: - prometheus_alertmanager - grafana + - name: openstack_infra_exporters + timeout: 600 + charts: + - prometheus_openstack_exporter + - name: openstack_infra_logging timeout: 600 charts: @@ -131,6 +136,16 @@ charts: ingress: public: false + prometheus_openstack_exporter: + chart_name: prometheus-openstack-exporter + release: prometheus-openstack-exporter + namespace: openstack + timeout: 300 + test: + enabled: false + timeout: 300 + output: false + grafana: chart_name: grafana release: prometheus-grafana