Add fluentd prometheus exporter for logging metrics

This adds templates for a prometheus exporter for fluentd to
adequately capture log metrics and fluentd service metrics for
consumption by prometheus

Change-Id: I6d6a8c2be07af819dc6d99b8ce5f1d4b635a69f0

fluent-logging/templates/deployment-fluentd.yaml

@@ -111,6 +111,8 @@ spec:
           ports:
             - name: forward
               containerPort: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: metrics
+              containerPort: {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
             - name: FLUENTD_PORT
               value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}

fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+COMMAND="${@:-start}"
+
+function start () {
+  exec go-wrapper run -scrape_uri "$FLUENTD_METRICS_HOST"
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-exporter-bin
+data:
+  fluentd-exporter.sh: |
+{{ tuple "bin/_fluentd-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml

@@ -0,0 +1,70 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.prometheus_fluentd_exporter }}
+
+{{ $fluentd_host := tuple "fluentd" "internal" "metrics" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+{{ $fluentd_metrics_path := "api/plugins.json" }}
+{{ $fluentd_metrics_host := printf "http://%s/%s" $fluentd_host $fluentd_metrics_path }}
+
+{{- $serviceAccountName := "prometheus-fluentd-exporter"}}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: prometheus-fluentd-exporter
+spec:
+  replicas: {{ .Values.pod.replicas.prometheus_fluentd_exporter }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      nodeSelector:
+        {{ .Values.labels.prometheus_fluentd_exporter.node_selector_key }}: {{ .Values.labels.prometheus_fluentd_exporter.node_selector_value }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_fluentd_exporter.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: fluentd-exporter
+          image: {{ .Values.images.tags.prometheus_fluentd_exporter }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.prometheus_fluentd_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentd-exporter.sh
+            - start
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.network.prometheus_fluentd_exporter.port }}
+          env:
+            - name: FLUENTD_METRICS_HOST
+              value: {{ $fluentd_metrics_host }}
+          volumeMounts:
+            - name: fluentd-exporter-bin
+              mountPath: /tmp/fluentd-exporter.sh
+              subPath: fluentd-exporter.sh
+              readOnly: true
+      volumes:
+        - name: fluentd-exporter-bin
+          configMap:
+            name: fluentd-exporter-bin
+            defaultMode: 0555
+{{- end }}

fluent-logging/templates/monitoring/prometheus/exporter-service.yaml

@@ -0,0 +1,37 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd_exporter }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "prometheus_fluentd_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "prometheus_fluentd_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+{{- if .Values.monitoring.prometheus.enabled }}
+{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
+{{- end }}
+spec:
+  ports:
+  - name: metrics
+    port: {{ .Values.network.prometheus_fluentd_exporter.port }}
+  selector:
+{{ tuple $envAll "prometheus_fluentd_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

fluent-logging/templates/service-fluentd.yaml

@@ -24,10 +24,12 @@ metadata:
 spec:
   ports:
   - name: forward
-    port:  {{ .Values.network.fluentd.port }}
+    port:  {{ .Values.network.fluentd.port.service }}
     {{ if .Values.network.fluentd.node_port.enabled }}
     nodePort: {{ .Values.network.fluentd.node_port.port }}
     {{ end }}
+  - name: metrics
+    port:  {{ .Values.network.fluentd.port.metrics }}
   selector:
 {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   {{ if .Values.network.fluentd.node_port.enabled }}

fluent-logging/values.yaml

@@ -25,11 +25,15 @@ labels:
   fluentbit:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  prometheus_fluentd_exporter:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 images:
   tags:
     fluentbit: docker.io/fluent/fluent-bit:0.12.9
     fluentd: docker.io/kolla/ubuntu-source-fluentd:ocata
+    prometheus_fluentd_exporter: docker.io/srwilkers/fluentd_exporter:v0.1
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1
     helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     image_repo_sync: docker.io/docker:17.07.0
@@ -69,6 +73,10 @@ dependencies:
       endpoint: internal
     - service: fluentd
       endpoint: internal
+  prometheus_fluentd_exporter:
+    services:
+    - service: fluentd
+      endpoint: internal
 
 conditional_dependencies:
   local_image_registry:
@@ -118,6 +126,11 @@ conf:
         Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
         Time_Keep: On
   td_agent:
+    - metrics_agent:
+        header: source
+        type: monitor_agent
+        bind: 0.0.0.0
+        port: 24220
     - fluentbit_forward:
         header: source
         type: forward
@@ -139,6 +152,10 @@ conf:
         max_retry_wait: 300
         disable_retry_limit: ""
         num_threads: 8
+  fluentd_exporter:
+    log:
+      format: "logger:stdout?json=true"
+      level: "info"
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -194,13 +211,38 @@ endpoints:
     port:
       service:
         default: 24224
+      metrics:
+        default: 24220
+  prometheus_fluentd_exporter:
+    namespace: null
+    hosts:
+      default: fluentd-exporter
+    host_fqdn_override:
+      default: null
+    path:
+      default: /metrics
+    scheme:
+      default: 'http'
+    port:
+      metrics:
+        default: 9309
+
+monitoring:
+  prometheus:
+    enabled: false
+    fluentd_exporter:
+      scrape: true
 
 network:
   fluentd:
     node_port:
       enabled: false
       port: 32329
-    port: 24224
+    port:
+      service: 24224
+      metrics: 24220
+  prometheus_fluentd_exporter:
+    port: 9309
 
 pod:
   affinity:
@@ -226,8 +268,11 @@ pod:
     termination_grace_period:
       fluentd:
         timeout: 30
+      prometheus_fluentd_exporter:
+        timeout: 30
   replicas:
     fluentd: 3
+    prometheus_fluentd_exporter: 1
   resources:
     fluentbit:
       enabled: false
@@ -245,6 +290,13 @@ pod:
       requests:
         memory: '128Mi'
         cpu: '500m'
+    prometheus_fluentd_exporter:
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+      requests:
+        memory: "128Mi"
+        cpu: "500m"
     jobs:
       image_repo_sync:
         requests:
@@ -275,5 +327,10 @@ manifests:
   daemonset_fluentbit: true
   job_image_repo_sync: true
   helm_tests: true
+  monitoring:
+    prometheus:
+      configmap_bin: true
+      deployment_exporter: true
+      service_exporter: true
   secret_elasticsearch: true
   service_fluentd: true

tools/gate/chart-deploys/default.yaml

@@ -216,6 +216,11 @@ charts:
       enabled: true
       timeout: 600
       output: false
+    values:
+      monitoring:
+        prometheus:
+          enabled: true
+
 
   kibana:
     chart_name: kibana