Add fluentbit sidecars to ceph mon and ceph osd charts

This adds fluentbit sidecar containers to the ceph mon and ceph osd charts, allowing for the gathering of ceph logs that arent sent to stdout and stderr. This sidecar container mounts a shared directory on /var/log/ceph, and fluentbit is configured to tail the log files in this directory and tag the event messages appropriately before forwarding them to fluentd. This behavior can be toggled on or off via values, and its proposed to be disabled by default Change-Id: Ia2c944a68f9e7d831ac0fe11ed09468d0dc9ce71
2018-06-15 15:00:40 -05:00 · 2018-06-15 15:00:40 -05:00 · b9209c47bf
commit b9209c47bf
parent 6f590a8d52
14 changed files with 337 additions and 8 deletions
--- a/ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl
+++ b/ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl
@ -0,0 +1,19 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
--- a/ceph-mon/templates/configmap-bin.yaml
+++ b/ceph-mon/templates/configmap-bin.yaml
@ -54,4 +54,8 @@ data:
  moncheck-reap-zombies.py: |
 {{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}
--- a/ceph-mon/templates/configmap-etc.yaml
+++ b/ceph-mon/templates/configmap-etc.yaml
@ -38,6 +38,14 @@ limitations under the License.
 {{- .Values.network.public | set .Values.conf.ceph.osd "public_network" | quote | trunc 0 -}}
 {{- end -}}

+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@ -46,7 +54,12 @@ metadata:
 data:
  ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- end }}
--- a/ceph-mon/templates/daemonset-mon.yaml
+++ b/ceph-mon/templates/daemonset-mon.yaml
@ -179,7 +179,33 @@ spec:
            - name: pod-run
              mountPath: /run
              readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-mon-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
      volumes:
+        - name: varlog
+          emptyDir: {}
        - name: ceph-mon-bin
          configMap:
            name: ceph-mon-bin
--- a/ceph-mon/templates/utils/_to_fluentbit_conf.tpl
+++ b/ceph-mon/templates/utils/_to_fluentbit_conf.tpl
@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-mon values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
--- a/ceph-mon/values.yaml
+++ b/ceph-mon/values.yaml
@ -23,6 +23,7 @@ images:
    ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
    ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
    ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
    image_repo_sync: docker.io/docker:17.07.0
  local_registry:
@ -65,6 +66,13 @@ pod:
      limits:
        memory: "50Mi"
        cpu: "500m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
    jobs:
      bootstrap:
        limits:
@ -151,6 +159,37 @@ conf:
  storage:
    mon:
      directory: /var/lib/openstack-helm/ceph/mon
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-mon logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-mon.*
+        Path: /var/log/ceph/*.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false

 dependencies:
  dynamic:
@ -250,6 +289,22 @@ endpoints:
    port:
      mon:
        default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220

 manifests:
  configmap_bin: true
--- a/ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl
+++ b/ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl
@ -0,0 +1,20 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
--- a/ceph-osd/templates/configmap-bin.yaml
+++ b/ceph-osd/templates/configmap-bin.yaml
@ -40,4 +40,8 @@ data:
 {{ tuple "bin/osd/_stop.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
  init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}
--- a/ceph-osd/templates/configmap-etc.yaml
+++ b/ceph-osd/templates/configmap-etc.yaml
@ -36,6 +36,14 @@ limitations under the License.
 {{- .Values.network.public | set .Values.conf.ceph.osd "public_network" | quote | trunc 0 -}}
 {{- end -}}

+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@ -44,7 +52,12 @@ metadata:
 data:
  ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}
--- a/ceph-osd/templates/daemonset-osd.yaml
+++ b/ceph-osd/templates/daemonset-osd.yaml
@ -75,7 +75,8 @@ spec:
            privileged: true
          env:
          # NOTE(portdirect): These environment variables will be populated
-          # dynamicly at the point of render.
+          # dynamicly at the point of render and added to all containers in the
+          # pod
          # - name: JOURNAL_LOCATION
          #   value: /var/lib/openstack-helm/ceph/osd/journal-one
          # - name: STORAGE_LOCATION
@ -210,6 +211,30 @@ spec:
            - name: journal
              mountPath: /var/lib/ceph/journal
              readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-osd-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
      volumes:
        - name: devices
          hostPath:
@ -219,6 +244,8 @@ spec:
            path: /run/lvm
        - name: pod-var-lib-ceph
          emptyDir: {}
+        - name: varlog
+          emptyDir: {}
        - name: pod-run
          emptyDir:
            medium: "Memory"
--- a/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl
+++ b/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl
@ -217,12 +217,16 @@ limitations under the License.
    {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
    {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}

-    {{/* set container name
-    assume not more than one container is defined */}}
-    {{- $container := first $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{/* set container names and add to the list of containers for the pod */}}
+    {{- $_ := set $context.Values "__containers_list" ( list ) }}
+    {{- range $container := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{- if eq $container.name "osd-pod" }}
    {{- $_ := set $container "name" $current_dict.dns_1123_name }}
-    {{- $cont_list := list $container }}
-    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $cont_list }}
+    {{- end }}
+    {{- $__containers_list := append $context.Values.__containers_list $container }}
+    {{- $_ := set $context.Values "__containers_list" $__containers_list }}
+    {{- end }}
+    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $context.Values.__containers_list }}

    {{/* cross-reference configmap name to container volume definitions */}}
    {{- $_ := set $context.Values "__volume_list" list }}
@ -310,6 +314,9 @@ limitations under the License.
  {{- $_ := set $context.Values "__tmpYAMLcontainers" list }}
  {{- range $podContainer := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
    {{- $_ := set $context.Values "_tmpYAMLcontainer" $podContainer }}
+    {{- if empty $context.Values._tmpYAMLcontainer.env }}
+    {{- $_ := set $context.Values._tmpYAMLcontainer "env" ( list ) }}
+    {{- end }}
    {{ $containerEnv := prepend (prepend (prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "JOURNAL_TYPE" "value" $v.journal.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "JOURNAL_LOCATION" "value" $v.journal.location) }}
    {{- $localInitContainerEnv := omit $context.Values._tmpYAMLcontainer "env" }}
    {{- $_ := set $localInitContainerEnv "env" $containerEnv }}
--- a/ceph-osd/templates/utils/_to_fluentbit_conf.tpl
+++ b/ceph-osd/templates/utils/_to_fluentbit_conf.tpl
@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-osd values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@ -17,6 +17,7 @@ images:
  tags:
    ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
    ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
    image_repo_sync: docker.io/docker:17.07.0
  local_registry:
@ -50,6 +51,13 @@ pod:
      limits:
        memory: "1024Mi"
        cpu: "1000m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
    jobs:
      image_repo_sync:
        requests:
@ -122,6 +130,37 @@ conf:
 #                 journal:
 #                   type: directory
 #                   location: /var/lib/openstack-helm/ceph/osd/journal-three
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-osd logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-osd.*
+        Path: /var/log/ceph/ceph-osd.**.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph-osd.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false

 dependencies:
  dynamic:
@ -182,6 +221,22 @@ endpoints:
    port:
      mon:
        default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220

 manifests:
  configmap_bin: true
--- a/tools/deployment/armada/multinode/armada-ceph.yaml
+++ b/tools/deployment/armada/multinode/armada-ceph.yaml
@ -73,6 +73,8 @@ data:
        namespace: ceph
      ceph_mon:
        namespace: ceph
+      fluentd:
+        namespace: osh-infra
    network:
      public: ${CEPH_NETWORK}
      cluster: ${CEPH_NETWORK}
@ -109,6 +111,9 @@ data:
            journal:
              type: directory
              location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
  source:
    type: local
    location: ${OSH_PATH}
@ -141,6 +146,8 @@ data:
        namespace: ceph
      ceph_mon:
        namespace: ceph
+      fluentd:
+        namespace: osh-infra
    network:
      public: ${CEPH_NETWORK}
      cluster: ${CEPH_NETWORK}
@ -177,6 +184,9 @@ data:
            journal:
              type: directory
              location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
  source:
    type: local
    location: ${OSH_PATH}