Add elasticsearch exporter

This adds a chart for an elasticsearch exporter to expose metrics for prometheus. This also moves the exporter to be included as part of the chart it's meant to target as opposed to its own separate chart Change-Id: I491f4d1efba633827d8a6255218daeb9d427f922
2017-11-28 18:55:41 -06:00 · 2017-11-28 18:55:41 -06:00 · 094e0103a9
commit 094e0103a9
parent 61792ba9af
7 changed files with 231 additions and 0 deletions
--- a/elasticsearch/templates/monitoring/prometheus/bin/_elasticsearch-exporter.sh.tpl
+++ b/elasticsearch/templates/monitoring/prometheus/bin/_elasticsearch-exporter.sh.tpl
@ -0,0 +1,33 @@
+#!/bin/sh
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+COMMAND="${@:-start}"
+
+function start () {
+  exec /bin/elasticsearch_exporter \
+        -es.uri=$ELASTICSEARCH_URI \
+        -es.all={{ .Values.conf.prometheus_elasticsearch_exporter.es.all | quote }} \
+        -es.timeout={{ .Values.conf.prometheus_elasticsearch_exporter.es.timeout }} \
+        -web.telemetry-path={{ .Values.endpoints.prometheus_elasticsearch_exporter.path.default }}
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND
--- a/elasticsearch/templates/monitoring/prometheus/exporter-configmap-bin.yaml
+++ b/elasticsearch/templates/monitoring/prometheus/exporter-configmap-bin.yaml
@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.configmap_bin_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-exporter-bin
+data:
+  elasticsearch-exporter.sh: |
+{{ tuple "bin/_elasticsearch-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
--- a/elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml
+++ b/elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml
@ -0,0 +1,80 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- if .Values.images.local_registry.active -}}
+{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.prometheus_elasticsearch_exporter .Values.conditional_dependencies.local_image_registry) -}}
+{{- else -}}
+{{- $_ := set .Values "pod_dependency" .Values.dependencies.prometheus_elasticsearch_exporter -}}
+{{- end -}}
+
+{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
+
+{{- $serviceAccountName := "prometheus-elasticsearch-exporter" }}
+{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: prometheus-elasticsearch-exporter
+spec:
+  replicas: {{ .Values.pod.replicas.prometheus_elasticsearch_exporter }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      nodeSelector:
+        {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_elasticsearch_exporter.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: elasticsearch-exporter
+{{ tuple $envAll "prometheus_elasticsearch_exporter" | include "helm-toolkit.snippets.image" | indent 10 }}
+          command:
+            - /tmp/elasticsearch-exporter.sh
+            - start
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/elasticsearch-exporter.sh
+                  - stop
+{{ tuple $envAll $envAll.Values.pod.resources.exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: ELASTICSEARCH_URI
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: ELASTICSEARCH_URI
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.network.prometheus_elasticsearch_exporter.port }}
+          volumeMounts:
+            - name: elasticsearch-exporter-bin
+              mountPath: /tmp/elasticsearch-exporter.sh
+              subPath: elasticsearch-exporter.sh
+              readOnly: true
+      volumes:
+        - name: elasticsearch-exporter-bin
+          configMap:
+            name: elasticsearch-exporter-bin
+            defaultMode: 0555
+{{- end }}
--- a/elasticsearch/templates/monitoring/prometheus/exporter-service.yaml
+++ b/elasticsearch/templates/monitoring/prometheus/exporter-service.yaml
@ -0,0 +1,37 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.elasticsearch_exporter }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "prometheus_elasticsearch_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "elasticsearch-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+{{- if .Values.monitoring.prometheus.enabled }}
+{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
+{{- end }}
+spec:
+  ports:
+  - name: metrics
+    port: {{ .Values.network.prometheus_elasticsearch_exporter.port }}
+  selector:
+{{ tuple $envAll "elasticsearch" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}
--- a/elasticsearch/templates/secret-admin-creds.yaml
+++ b/elasticsearch/templates/secret-admin-creds.yaml
@ -17,6 +17,11 @@ limitations under the License.
 {{- if .Values.manifests.secret_admin }}
 {{- $envAll := . }}
 {{- $secretName := index $envAll.Values.secrets.elasticsearch.user }}
+
+{{- $elasticsearch_user := .Values.endpoints.elasticsearch.auth.admin.username }}
+{{- $elasticsearch_password := .Values.endpoints.elasticsearch.auth.admin.password }}
+{{- $elasticsearch_host := tuple "elasticsearch" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{- $elasticsearch_uri := printf "http://%s:%s@%s" $elasticsearch_user $elasticsearch_password $elasticsearch_host }}
 ---
 apiVersion: v1
 kind: Secret
@ -26,4 +31,5 @@ type: Opaque
 data:
  ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
  ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+  ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }}
 {{- end }}
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@ -23,6 +23,7 @@ images:
    curator:  docker.io/bobrik/curator:5.2.0
    elasticsearch: docker.io/elasticsearch:5.6.4
    helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
+    prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1
    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
    snapshot_repository: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
    image_repo_sync: docker.io/docker:17.07.0
@ -44,6 +45,10 @@ dependencies:
    services: null
  elasticsearch_data:
    services: null
+  prometheus_elasticsearch_exporter:
+    services:
+      - service: elasticsearch
+        endpoint: internal
  curator:
    services: null
  image_repo_sync:
@ -89,6 +94,8 @@ pod:
        timeout: 600
      client:
        timeout: 600
+      prometheus_elasticsearch_exporter:
+        timeout: 600
  mounts:
    elasticsearch:
      elasticsearch:
@ -115,6 +122,13 @@ pod:
      limits:
        memory: "1024Mi"
        cpu: "2000m"
+    prometheus_elasticsearch_exporter:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
    jobs:
      curator:
        requests:
@ -265,6 +279,10 @@ conf:
    override:
    prefix:
    append:
+  prometheus_elasticsearch_exporter:
+    es:
+      all: true
+      timeout: 20s

 endpoints:
  cluster_domain_suffix: cluster.local
@ -293,6 +311,25 @@ endpoints:
        default: 80
      discovery:
        default: 9300
+  prometheus_elasticsearch_exporter:
+    namespace: null
+    hosts:
+      default: elasticsearch-exporter
+    host_fqdn_override:
+      default: null
+    path:
+      default: /metrics
+    scheme:
+      default: 'http'
+    port:
+      metrics:
+        default: 9108
+
+monitoring:
+  prometheus:
+    enabled: false
+    elasticsearch_exporter:
+      scrape: true

 network:
  client:
@ -310,6 +347,8 @@ network:
    node_port:
      enabled: false
      port: 30931
+  prometheus_elasticsearch_exporter:
+    port: 9108

 storage:
  elasticsearch:
@ -341,6 +380,12 @@ manifests:
  helm_tests: true
  pvc_snapshots: true
  secret_admin: true
+  monitoring:
+    prometheus:
+      configmap_bin_exporter: true
+      deployment_exporter: true
+      service_exporter: true
+  pvc_snapshots: true
  service_data: true
  service_discovery: true
  service_logging: true
--- a/tools/gate/chart-deploys/default.yaml
+++ b/tools/gate/chart-deploys/default.yaml
@ -203,6 +203,9 @@ charts:
          storage_class: openstack-helm-bootstrap
        filesystem_repository:
          storage_class: openstack-helm-bootstrap
+      monitoring:
+        prometheus:
+          enabled: true

  fluent_logging:
    chart_name: fluent-logging