Merge "Add fluentbit sidecars to ceph mon and ceph osd charts"

2018-06-19 23:53:24 +00:00 · 2018-06-19 23:53:24 +00:00 · ef03612edf
commit ef03612edf
parent c94042d619 b9209c47bf
14 changed files with 337 additions and 8 deletions
--- a/ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl
+++ b/ceph-mon/templates/bin/mon/_fluentbit-sidecar.sh.tpl
@ -0,0 +1,19 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
--- a/ceph-mon/templates/configmap-bin.yaml
+++ b/ceph-mon/templates/configmap-bin.yaml
@ -54,4 +54,8 @@ data:
  moncheck-reap-zombies.py: |
 {{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/mon/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}
--- a/ceph-mon/templates/configmap-etc.yaml
+++ b/ceph-mon/templates/configmap-etc.yaml
@ -38,6 +38,14 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}

+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@ -46,7 +54,12 @@ metadata:
 data:
  ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-mon.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- end }}
--- a/ceph-mon/templates/daemonset-mon.yaml
+++ b/ceph-mon/templates/daemonset-mon.yaml
@ -184,7 +184,33 @@ spec:
            - name: pod-run
              mountPath: /run
              readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-mon-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-mon-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
      volumes:
+        - name: varlog
+          emptyDir: {}
        - name: ceph-mon-bin
          configMap:
            name: ceph-mon-bin
--- a/ceph-mon/templates/utils/_to_fluentbit_conf.tpl
+++ b/ceph-mon/templates/utils/_to_fluentbit_conf.tpl
@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-mon values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-mon.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
--- a/ceph-mon/values.yaml
+++ b/ceph-mon/values.yaml
@ -23,6 +23,7 @@ images:
    ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
    ceph_mon: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
    ceph_mon_check: 'docker.io/port/ceph-config-helper:v1.10.3'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
    image_repo_sync: docker.io/docker:17.07.0
  local_registry:
@ -65,6 +66,13 @@ pod:
      limits:
        memory: "50Mi"
        cpu: "500m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
    jobs:
      bootstrap:
        limits:
@ -151,6 +159,37 @@ conf:
  storage:
    mon:
      directory: /var/lib/openstack-helm/ceph/mon
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-mon logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-mon.*
+        Path: /var/log/ceph/*.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false

 dependencies:
  dynamic:
@ -250,6 +289,22 @@ endpoints:
    port:
      mon:
        default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220

 manifests:
  configmap_bin: true
--- a/ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl
+++ b/ceph-osd/templates/bin/osd/_fluentbit-sidecar.sh.tpl
@ -0,0 +1,20 @@
+#!/bin/sh
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf
--- a/ceph-osd/templates/configmap-bin.yaml
+++ b/ceph-osd/templates/configmap-bin.yaml
@ -40,4 +40,8 @@ data:
 {{ tuple "bin/osd/_stop.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
  init-dirs.sh: |
 {{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ if .Values.logging.fluentd }}
+  fluentbit-sidecar.sh: |
+{{ tuple "bin/osd/_fluentbit-sidecar.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{ end }}
 {{- end }}
--- a/ceph-osd/templates/configmap-etc.yaml
+++ b/ceph-osd/templates/configmap-etc.yaml
@ -36,6 +36,14 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}

+{{- if not (has "fluentd_output" .Values.conf.fluentbit) -}}
+{{- $fluentd_host := tuple "fluentd" "internal" $envAll | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }}
+{{- $fluentd_port := tuple "fluentd" "internal" "service" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $fluentd_output := dict "header" "output" "Name" "forward" "Match" "*" "Host" $fluentd_host "Port" $fluentd_port -}}
+{{- $_ := set .Values "__fluentbit_config" ( list $fluentd_output) -}}
+{{- $__fluentbit_config := append .Values.conf.fluentbit .Values.__fluentbit_config -}}
+{{- $_ := set .Values.conf "fluentbit" $__fluentbit_config -}}
+{{- end -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@ -44,7 +52,12 @@ metadata:
 data:
  ceph.conf: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
-
+{{ if .Values.logging.fluentd }}
+  fluent-bit.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.fluentbit | indent 4 }}
+  parsers.conf: |
+{{ include "ceph-osd.utils.to_fluentbit_conf" .Values.conf.parsers | indent 4 }}
+{{ end }}
 {{- end }}
 {{- end }}
 {{- if .Values.manifests.configmap_etc }}
--- a/ceph-osd/templates/daemonset-osd.yaml
+++ b/ceph-osd/templates/daemonset-osd.yaml
@ -80,7 +80,8 @@ spec:
            privileged: true
          env:
          # NOTE(portdirect): These environment variables will be populated
-          # dynamicly at the point of render.
+          # dynamicly at the point of render and added to all containers in the
+          # pod
          # - name: JOURNAL_LOCATION
          #   value: /var/lib/openstack-helm/ceph/osd/journal-one
          # - name: STORAGE_LOCATION
@ -215,6 +216,30 @@ spec:
            - name: journal
              mountPath: /var/lib/ceph/journal
              readOnly: false
+            - name: varlog
+              mountPath: /var/log/ceph
+        {{ if .Values.logging.fluentd }}
+        - name: fluentbit-sidecar
+{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/fluentbit-sidecar.sh
+          volumeMounts:
+            - name: ceph-osd-bin
+              mountPath: /tmp/fluentbit-sidecar.sh
+              subPath: fluentbit-sidecar.sh
+              readOnly: true
+            - name: varlog
+              mountPath: /var/log/ceph
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/fluent-bit.conf
+              subPath: fluent-bit.conf
+              readOnly: true
+            - name: ceph-osd-etc
+              mountPath: /fluent-bit/etc/parsers.conf
+              subPath: parsers.conf
+              readOnly: true
+        {{ end }}
      volumes:
        - name: devices
          hostPath:
@ -224,6 +249,8 @@ spec:
            path: /run/lvm
        - name: pod-var-lib-ceph
          emptyDir: {}
+        - name: varlog
+          emptyDir: {}
        - name: pod-run
          emptyDir:
            medium: "Memory"
--- a/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl
+++ b/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl
@ -217,12 +217,16 @@ limitations under the License.
    {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }}
    {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }}

-    {{/* set container name
-    assume not more than one container is defined */}}
-    {{- $container := first $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{/* set container names and add to the list of containers for the pod */}}
+    {{- $_ := set $context.Values "__containers_list" ( list ) }}
+    {{- range $container := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
+    {{- if eq $container.name "osd-pod" }}
    {{- $_ := set $container "name" $current_dict.dns_1123_name }}
-    {{- $cont_list := list $container }}
-    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $cont_list }}
+    {{- end }}
+    {{- $__containers_list := append $context.Values.__containers_list $container }}
+    {{- $_ := set $context.Values "__containers_list" $__containers_list }}
+    {{- end }}
+    {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "containers" $context.Values.__containers_list }}

    {{/* cross-reference configmap name to container volume definitions */}}
    {{- $_ := set $context.Values "__volume_list" list }}
@ -310,6 +314,9 @@ limitations under the License.
  {{- $_ := set $context.Values "__tmpYAMLcontainers" list }}
  {{- range $podContainer := $context.Values.__daemonset_yaml.spec.template.spec.containers }}
    {{- $_ := set $context.Values "_tmpYAMLcontainer" $podContainer }}
+    {{- if empty $context.Values._tmpYAMLcontainer.env }}
+    {{- $_ := set $context.Values._tmpYAMLcontainer "env" ( list ) }}
+    {{- end }}
    {{ $containerEnv := prepend (prepend (prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "JOURNAL_TYPE" "value" $v.journal.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "JOURNAL_LOCATION" "value" $v.journal.location) }}
    {{- $localInitContainerEnv := omit $context.Values._tmpYAMLcontainer "env" }}
    {{- $_ := set $localInitContainerEnv "env" $containerEnv }}
--- a/ceph-osd/templates/utils/_to_fluentbit_conf.tpl
+++ b/ceph-osd/templates/utils/_to_fluentbit_conf.tpl
@ -0,0 +1,38 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+# This function generates fluentbit configuration files with entries in the
+# ceph-osd values.yaml.  It results in a configuration section with the
+# following format (for as many key/value pairs defined in values for a section):
+# [HEADER]
+#     key value
+#     key value
+#     key value
+# The configuration schema can be found here:
+# http://fluentbit.io/documentation/0.12/configuration/schema.html
+
+{{- define "ceph-osd.utils.to_fluentbit_conf" -}}
+{{- range $values := . -}}
+{{- range $section := . -}}
+{{- $header := pick . "header" -}}
+{{- $config := omit . "header" }}
+[{{$header.header | upper }}]
+{{range $key, $value := $config -}}
+{{ $key | indent 4 }} {{ $value }}
+{{end -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@ -17,6 +17,7 @@ images:
  tags:
    ceph_osd: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
    ceph_bootstrap: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    fluentbit: docker.io/fluent/fluent-bit:0.12.14
    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
    image_repo_sync: docker.io/docker:17.07.0
  local_registry:
@ -50,6 +51,13 @@ pod:
      limits:
        memory: "1024Mi"
        cpu: "1000m"
+    fluentbit:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
    jobs:
      image_repo_sync:
        requests:
@ -122,6 +130,37 @@ conf:
 #                 journal:
 #                   type: directory
 #                   location: /var/lib/openstack-helm/ceph/osd/journal-three
+  fluentbit:
+    - service:
+        header: service
+        Flush: 30
+        Daemon: Off
+        Log_Level: info
+        Parsers_File: parsers.conf
+    - ceph_tail:
+        # NOTE(srwilkers): Despite being exposed, these values should not be
+        # modified, as the ceph-osd logs are always placed here
+        header: input
+        Name: tail
+        Tag: ceph-osd.*
+        Path: /var/log/ceph/ceph-osd.**.log
+        Parser: syslog
+        DB: /var/log/ceph/ceph-osd.db
+        Mem_Buf_Limit: 5MB
+        Refresh_Interval: 10s
+  parsers:
+    - syslog:
+        header: parser
+        Name: syslog
+        Format: regex
+        Regex: '^(?<time>.*[0-9]{2}:[0-9]{2}:[0-9]{2}) (?<host>[^ ]*) (?<app>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? (?<log>.+)$'
+        Time_Key: time
+        Time_Format: "%Y-%m-%dT%H:%M:%S.%L"
+        Time_Keep: On
+        Types: "pid:integer"
+
+logging:
+  fluentd: false

 dependencies:
  dynamic:
@ -182,6 +221,22 @@ endpoints:
    port:
      mon:
        default: 6789
+  fluentd:
+    namespace: null
+    name: fluentd
+    hosts:
+      default: fluentd-logging
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      service:
+        default: 24224
+      metrics:
+        default: 24220

 manifests:
  configmap_bin: true
--- a/tools/deployment/armada/multinode/armada-ceph.yaml
+++ b/tools/deployment/armada/multinode/armada-ceph.yaml
@ -73,6 +73,8 @@ data:
        namespace: ceph
      ceph_mon:
        namespace: ceph
+      fluentd:
+        namespace: osh-infra
    network:
      public: ${CEPH_NETWORK}
      cluster: ${CEPH_NETWORK}
@ -109,6 +111,9 @@ data:
            journal:
              type: directory
              location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
  source:
    type: local
    location: ${OSH_PATH}
@ -141,6 +146,8 @@ data:
        namespace: ceph
      ceph_mon:
        namespace: ceph
+      fluentd:
+        namespace: osh-infra
    network:
      public: ${CEPH_NETWORK}
      cluster: ${CEPH_NETWORK}
@ -177,6 +184,9 @@ data:
            journal:
              type: directory
              location: /var/lib/openstack-helm/ceph/osd/journal-one
+    logging:
+      fluentd:
+        enabled: true
  source:
    type: local
    location: ${OSH_PATH}