From abb5e0f713aa07b8c90cc5e593135ca338c5ff6f Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 8 Mar 2019 14:38:48 -0600 Subject: [PATCH] Separate fluentbit and fluentd charts This begins to split the fluent-logging chart into two separate charts, one for fluentbit and one for fluentd. This is to help isolate each chart and its dependencies better, and to treat each service as its own entity. This also moves the job for creating Elasticsearch templates to the Elasticsearch chart, as the elasticsearch chart should have ownership of creating the templates for its indices. This also performs some general cleanup of values keys that are not currently used Change-Id: I827277d5faa62b8b59c5960330703d23c297ca47 Signed-off-by: Steve Wilkerson --- doc/source/install/multinode.rst | 21 +- .../templates/bin/_create_template.sh.tpl | 0 .../templates/bin/_helm-tests.sh.tpl | 19 ++ .../configmap-bin-elasticsearch.yaml | 2 + .../templates/configmap-etc-templates.yaml | 22 +- .../templates/job-elasticsearch-template.yaml | 38 +-- elasticsearch/values.yaml | 45 +++ fluent-logging/README.rst | 30 -- .../templates/bin/_helm-tests.sh.tpl | 41 --- fluent-logging/templates/pod-helm-tests.yaml | 75 ----- fluentbit/Chart.yaml | 24 ++ .../requirements.yaml | 0 .../templates/bin/_fluent-bit.sh.tpl | 0 .../templates/configmap-bin.yaml | 8 +- .../templates/configmap-etc.yaml | 6 +- .../templates/daemonset-fluent-bit.yaml | 18 +- .../templates/job-image-repo-sync.yaml | 2 +- fluentbit/values.yaml | 256 ++++++++++++++++++ {fluent-logging => fluentd}/Chart.yaml | 3 +- fluentd/requirements.yaml | 18 ++ .../templates/bin/_fluentd.sh.tpl | 0 fluentd/templates/configmap-bin.yaml | 29 ++ fluentd/templates/configmap-etc.yaml | 27 ++ .../templates/deployment-fluentd.yaml | 16 +- fluentd/templates/job-image-repo-sync.yaml | 20 ++ .../prometheus/bin/_fluentd-exporter.sh.tpl | 0 .../prometheus/exporter-configmap-bin.yaml | 0 .../prometheus/exporter-deployment.yaml | 0 .../prometheus/exporter-service.yaml | 0 .../templates/network_policy.yaml | 0 .../templates/secret-elasticsearch-creds.yaml | 0 .../templates/service-fluentd.yaml | 0 {fluent-logging => fluentd}/values.yaml | 154 +---------- .../armada/manifests/armada-lma.yaml | 136 +++++++++- tools/deployment/common/fluentbit.sh | 29 ++ tools/deployment/multinode/125-fluentbit.sh | 1 + .../{130-fluent-logging.sh => 130-fluentd.sh} | 18 +- .../osh-infra-logging/060-fluentbit.sh | 1 + .../{060-fluent-logging.sh => 065-fluentd.sh} | 14 +- zuul.d/jobs.yaml | 6 +- 40 files changed, 688 insertions(+), 391 deletions(-) rename {fluent-logging => elasticsearch}/templates/bin/_create_template.sh.tpl (100%) rename fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml => elasticsearch/templates/configmap-etc-templates.yaml (54%) rename {fluent-logging => elasticsearch}/templates/job-elasticsearch-template.yaml (66%) delete mode 100644 fluent-logging/README.rst delete mode 100644 fluent-logging/templates/bin/_helm-tests.sh.tpl delete mode 100644 fluent-logging/templates/pod-helm-tests.yaml create mode 100644 fluentbit/Chart.yaml rename {fluent-logging => fluentbit}/requirements.yaml (100%) rename {fluent-logging => fluentbit}/templates/bin/_fluent-bit.sh.tpl (100%) rename {fluent-logging => fluentbit}/templates/configmap-bin.yaml (71%) rename {fluent-logging => fluentbit}/templates/configmap-etc.yaml (80%) rename {fluent-logging => fluentbit}/templates/daemonset-fluent-bit.yaml (90%) rename {fluent-logging => fluentbit}/templates/job-image-repo-sync.yaml (90%) create mode 100644 fluentbit/values.yaml rename {fluent-logging => fluentd}/Chart.yaml (93%) create mode 100644 fluentd/requirements.yaml rename {fluent-logging => fluentd}/templates/bin/_fluentd.sh.tpl (100%) create mode 100644 fluentd/templates/configmap-bin.yaml create mode 100644 fluentd/templates/configmap-etc.yaml rename {fluent-logging => fluentd}/templates/deployment-fluentd.yaml (92%) create mode 100644 fluentd/templates/job-image-repo-sync.yaml rename {fluent-logging => fluentd}/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl (100%) rename {fluent-logging => fluentd}/templates/monitoring/prometheus/exporter-configmap-bin.yaml (100%) rename {fluent-logging => fluentd}/templates/monitoring/prometheus/exporter-deployment.yaml (100%) rename {fluent-logging => fluentd}/templates/monitoring/prometheus/exporter-service.yaml (100%) rename fluent-logging/templates/network-policy.yaml => fluentd/templates/network_policy.yaml (100%) rename {fluent-logging => fluentd}/templates/secret-elasticsearch-creds.yaml (100%) rename {fluent-logging => fluentd}/templates/service-fluentd.yaml (100%) rename {fluent-logging => fluentd}/values.yaml (78%) create mode 100755 tools/deployment/common/fluentbit.sh create mode 120000 tools/deployment/multinode/125-fluentbit.sh rename tools/deployment/multinode/{130-fluent-logging.sh => 130-fluentd.sh} (75%) create mode 120000 tools/deployment/osh-infra-logging/060-fluentbit.sh rename tools/deployment/osh-infra-logging/{060-fluent-logging.sh => 065-fluentd.sh} (82%) diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 9246af02b..d06f002e4 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -210,10 +210,10 @@ Alternatively, this step can be performed by running the script directly: ./tools/deployment/multinode/120-elasticsearch.sh -Deploy Fluent-Logging -^^^^^^^^^^^^^^^^^^^^^ +Deploy Fluentbit +^^^^^^^^^^^^^^^^ -.. literalinclude:: ../../../tools/deployment/multinode/130-fluent-logging.sh +.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh :language: shell :lines: 1,17- @@ -221,4 +221,17 @@ Alternatively, this step can be performed by running the script directly: .. code-block:: shell - ./tools/deployment/multinode/130-fluent-logging.sh + ./tools/deployment/multinode/125-fluentbit.sh + +Deploy Fluentd +^^^^^^^^^^^^^^ + +.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh + :language: shell + :lines: 1,17- + +Alternatively, this step can be performed by running the script directly: + +.. code-block:: shell + + ./tools/deployment/multinode/130-fluentd.sh diff --git a/fluent-logging/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_create_template.sh.tpl rename to elasticsearch/templates/bin/_create_template.sh.tpl diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index 80c02a830..877659232 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -52,6 +52,24 @@ function check_snapshot_repositories () { {{ end }} } +{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} +# Tests whether elasticsearch has successfully generated the elasticsearch index mapping +# templates defined by values.yaml +function check_templates () { + {{ range $template, $fields := .Values.conf.templates }} + {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ + | python -c "import sys, json; print len(json.load(sys.stdin))") + if [ "${{$template}}_total_hits" -gt 0 ]; then + echo "PASS: Successful hits on {{$template}} template!" + else + echo "FAIL: No hits on query for {{$template}} template! Exiting"; + exit 1; + fi + {{ end }} +} +{{ end }} + function remove_test_index () { echo "Deleting index created for service testing" curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ @@ -63,4 +81,5 @@ create_test_index {{ if .Values.conf.elasticsearch.snapshots.enabled }} check_snapshot_repositories {{ end }} +check_templates remove_test_index diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml index 4f213835f..6627e2d2d 100644 --- a/elasticsearch/templates/configmap-bin-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -38,6 +38,8 @@ data: {{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} es-cluster-wait.sh: | {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_template.sh: | +{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml b/elasticsearch/templates/configmap-etc-templates.yaml similarity index 54% rename from fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml rename to elasticsearch/templates/configmap-etc-templates.yaml index b101d9fd5..5e2ea5732 100644 --- a/fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml +++ b/elasticsearch/templates/configmap-etc-templates.yaml @@ -1,11 +1,11 @@ {{/* -Copyright 2017-2018 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} -{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }} -{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} -{{- end -}} +{{- if .Values.manifests.configmap_etc_templates }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-templates-etc +type: Opaque +data: +{{ range $template, $fields := .Values.conf.templates }} + {{ $template }}.json: {{ toJson $fields | b64enc }} +{{ end }} +{{- end }} diff --git a/fluent-logging/templates/job-elasticsearch-template.yaml b/elasticsearch/templates/job-elasticsearch-template.yaml similarity index 66% rename from fluent-logging/templates/job-elasticsearch-template.yaml rename to elasticsearch/templates/job-elasticsearch-template.yaml index 5b18df715..d563d7ee1 100644 --- a/fluent-logging/templates/job-elasticsearch-template.yaml +++ b/elasticsearch/templates/job-elasticsearch-template.yaml @@ -14,26 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} +{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} -{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }} -{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }} +{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }} +{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }} -{{- $serviceAccountName := "fluent-logging-elasticsearch-template" }} -{{ tuple $envAll "elasticsearch_template" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- $serviceAccountName := "create-elasticsearch-templates" }} +{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job metadata: - name: elasticsearch-template + name: create-elasticsearch-templates annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} spec: template: metadata: labels: -{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: {{ dict "envAll" $envAll "application" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} @@ -41,11 +41,11 @@ spec: nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} initContainers: -{{ tuple $envAll "elasticsearch_template" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - - name: elasticsearch-template -{{ tuple $envAll "elasticsearch_template" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + - name: create-elasticsearch-templates +{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "elasticsearch_template" "container" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} env: - name: ELASTICSEARCH_HOST @@ -67,27 +67,27 @@ spec: volumeMounts: - name: pod-tmp mountPath: /tmp - - name: fluent-logging-bin + - name: elasticsearch-bin mountPath: /tmp/create_template.sh subPath: create_template.sh readOnly: true {{ range $template, $fields := .Values.conf.templates }} - - name: fluent-logging-etc + - name: elasticsearch-templates-etc mountPath: /tmp/{{$template}}.json subPath: {{$template}}.json readOnly: true {{ end }} -{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }} +{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }} volumes: - name: pod-tmp emptyDir: {} - - name: fluent-logging-bin + - name: elasticsearch-bin configMap: - name: fluent-logging-bin + name: elasticsearch-bin defaultMode: 0555 - - name: fluent-logging-etc + - name: elasticsearch-templates-etc secret: - secretName: fluent-logging-etc + secretName: elasticsearch-templates-etc defaultMode: 0444 -{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 8 }}{{ end }} +{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 57f397f14..1fe09a089 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -30,6 +30,7 @@ images: dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 snapshot_repository: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial es_cluster_wait: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial + elasticsearch_templates: docker.io/openstackhelm/heat:newton image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" local_registry: @@ -82,6 +83,10 @@ dependencies: services: - endpoint: internal service: elasticsearch + elasticsearch_templates: + services: null + jobs: + - elasticsearch-cluster-wait image_repo_sync: services: - endpoint: internal @@ -220,6 +225,8 @@ pod: mounts: elasticsearch: elasticsearch: + elasticsearch_templates: + elasticsearch_templates: resources: enabled: false apache_proxy: @@ -265,6 +272,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + elasticsearch_templates: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" image_repo_sync: requests: memory: "128Mi" @@ -635,6 +649,35 @@ conf: es: all: true timeout: 20s + templates: + fluent: + template: "logstash-*" + index_patterns: "logstash-*" + settings: + number_of_shards: 1 + mappings: + fluent: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false endpoints: cluster_domain_suffix: cluster.local @@ -771,11 +814,13 @@ manifests: configmap_bin_elasticsearch: true configmap_etc_curator: true configmap_etc_elasticsearch: true + configmap_etc_templates: true cron_curator: true deployment_client: true deployment_master: true ingress: true job_cluster_wait: true + job_elasticsearch_templates: true job_image_repo_sync: true job_snapshot_repository: true job_s3_user: true diff --git a/fluent-logging/README.rst b/fluent-logging/README.rst deleted file mode 100644 index c2f035db4..000000000 --- a/fluent-logging/README.rst +++ /dev/null @@ -1,30 +0,0 @@ -Fluentd-logging -=============== - -OpenStack-Helm defines a centralized logging mechanism to provide insight into -the state of the OpenStack services and infrastructure components as -well as underlying kubernetes platform. Among the requirements for a logging -platform, where log data can come from and where log data need to be delivered -are very variable. To support various logging scenarios, OpenStack-Helm should -provide a flexible mechanism to meet with certain operation needs. This chart -proposes fast and lightweight log forwarder and full featured log aggregator -complementing each other providing a flexible and reliable solution. Especially, -Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log -aggregator and processor. - - -Mechanism ---------- - -Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering, -aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on -each node and mounts the /var/lib/docker/containers directory. The Docker -container runtime engine directs events posted to stdout and stderr to this -directory on the host. Fluent-bit then forward the contents of that directory to -Fluentd. Fluentd runs as deployment at the designated nodes and expose service -for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to -the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs. -Fluentd will then forward the results to Elasticsearch and to optionally kafka. -Elasticsearch indexes the logs in a logstash-* index by default. kafka stores -the logs in a 'logs' topic by default. Any external tool can then consume the -'logs' topic. diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl deleted file mode 100644 index 24d94f11a..000000000 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex - -{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} -# Tests whether fluent-logging has successfully generated the elasticsearch index mapping -# templates defined by values.yaml -function check_templates () { - {{ range $template, $fields := .Values.conf.templates }} - {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ - | python -c "import sys, json; print len(json.load(sys.stdin))") - if [ "${{$template}}_total_hits" -gt 0 ]; then - echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!" - else - echo "FAIL: No hits on query for {{$template}} template! Exiting"; - exit 1; - fi - {{ end }} -} -{{ end }} - -{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} -check_templates -{{ end }} diff --git a/fluent-logging/templates/pod-helm-tests.yaml b/fluent-logging/templates/pod-helm-tests.yaml deleted file mode 100644 index 053e66a4e..000000000 --- a/fluent-logging/templates/pod-helm-tests.yaml +++ /dev/null @@ -1,75 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.helm_tests }} -{{- $envAll := . }} -{{- $esUserSecret := .Values.secrets.elasticsearch.user }} - -{{- $serviceAccountName := print .Release.Name "-test" }} -{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: v1 -kind: Pod -metadata: - name: "{{.Release.Name}}-test" - labels: -{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - annotations: - "helm.sh/hook": test-success - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: -{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} - restartPolicy: Never - initContainers: -{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} - containers: - - name: {{.Release.Name}}-helm-tests -{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - command: - - /tmp/helm-tests.sh - env: - - name: ELASTICSEARCH_USERNAME - valueFrom: - secretKeyRef: - name: {{ $esUserSecret }} - key: ELASTICSEARCH_USERNAME - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: {{ $esUserSecret }} - key: ELASTICSEARCH_PASSWORD - - name: ELASTICSEARCH_ENDPOINT - value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} - volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: fluent-logging-bin - mountPath: /tmp/helm-tests.sh - subPath: helm-tests.sh - readOnly: true - volumes: - - name: pod-tmp - emptyDir: {} - - name: fluent-logging-bin - configMap: - name: fluent-logging-bin - defaultMode: 0555 -{{- end }} diff --git a/fluentbit/Chart.yaml b/fluentbit/Chart.yaml new file mode 100644 index 000000000..7a88e5e0d --- /dev/null +++ b/fluentbit/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Fluentbit +name: fluentbit +version: 0.1.0 +home: http://www.fluentbit.io/ +sources: + - https://github.com/fluent/fluentbit + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/fluent-logging/requirements.yaml b/fluentbit/requirements.yaml similarity index 100% rename from fluent-logging/requirements.yaml rename to fluentbit/requirements.yaml diff --git a/fluent-logging/templates/bin/_fluent-bit.sh.tpl b/fluentbit/templates/bin/_fluent-bit.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_fluent-bit.sh.tpl rename to fluentbit/templates/bin/_fluent-bit.sh.tpl diff --git a/fluent-logging/templates/configmap-bin.yaml b/fluentbit/templates/configmap-bin.yaml similarity index 71% rename from fluent-logging/templates/configmap-bin.yaml rename to fluentbit/templates/configmap-bin.yaml index e331e36e1..ea63b42f9 100644 --- a/fluent-logging/templates/configmap-bin.yaml +++ b/fluentbit/templates/configmap-bin.yaml @@ -20,16 +20,10 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: fluent-logging-bin + name: fluentbit-bin data: - fluentd.sh: | -{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} fluent-bit.sh: | {{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - helm-tests.sh: | -{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - create_template.sh: | -{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluentbit/templates/configmap-etc.yaml similarity index 80% rename from fluent-logging/templates/configmap-etc.yaml rename to fluentbit/templates/configmap-etc.yaml index fae65dcac..dcafb1eec 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluentbit/templates/configmap-etc.yaml @@ -20,13 +20,9 @@ limitations under the License. apiVersion: v1 kind: Secret metadata: - name: fluent-logging-etc + name: fluentbit-etc type: Opaque data: fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }} parsers.conf: {{ .Values.conf.parsers.template | b64enc }} - fluent.conf: {{ .Values.conf.fluentd.template | b64enc }} -{{ range $template, $fields := .Values.conf.templates }} - {{ $template }}.json: {{ toJson $fields | b64enc }} -{{ end }} {{- end }} diff --git a/fluent-logging/templates/daemonset-fluent-bit.yaml b/fluentbit/templates/daemonset-fluent-bit.yaml similarity index 90% rename from fluent-logging/templates/daemonset-fluent-bit.yaml rename to fluentbit/templates/daemonset-fluent-bit.yaml index abe9b841f..f9c59e371 100644 --- a/fluent-logging/templates/daemonset-fluent-bit.yaml +++ b/fluentbit/templates/daemonset-fluent-bit.yaml @@ -95,7 +95,7 @@ spec: configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: -{{ dict "envAll" $envAll "application" "daemon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} +{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} {{ if $envAll.Values.pod.tolerations.fluentbit.enabled }} {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} @@ -112,7 +112,7 @@ spec: - name: fluentbit {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "daemon" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/fluent-bit.sh env: @@ -123,7 +123,7 @@ spec: volumeMounts: - name: pod-tmp mountPath: /tmp - - name: fluent-logging-bin + - name: fluentbit-bin mountPath: /tmp/fluent-bit.sh subPath: fluent-bit.sh readOnly: true @@ -133,11 +133,11 @@ spec: - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - - name: fluent-logging-etc + - name: fluentbit-etc mountPath: /fluent-bit/etc/fluent-bit.conf subPath: fluent-bit.conf readOnly: true - - name: fluent-logging-etc + - name: fluentbit-etc mountPath: /fluent-bit/etc/parsers.conf subPath: parsers.conf readOnly: true @@ -151,13 +151,13 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: fluent-logging-bin + - name: fluentbit-bin configMap: - name: fluent-logging-bin + name: fluentbit-bin defaultMode: 0555 - - name: fluent-logging-etc + - name: fluentbit-etc secret: - secretName: fluent-logging-etc + secretName: fluentbit-etc defaultMode: 0444 {{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/fluent-logging/templates/job-image-repo-sync.yaml b/fluentbit/templates/job-image-repo-sync.yaml similarity index 90% rename from fluent-logging/templates/job-image-repo-sync.yaml rename to fluentbit/templates/job-image-repo-sync.yaml index 02c56ab7e..0916c7107 100644 --- a/fluent-logging/templates/job-image-repo-sync.yaml +++ b/fluentbit/templates/job-image-repo-sync.yaml @@ -15,6 +15,6 @@ limitations under the License. */}} {{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} -{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluent-logging" -}} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}} {{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} {{- end }} diff --git a/fluentbit/values.yaml b/fluentbit/values.yaml new file mode 100644 index 000000000..ac30a603b --- /dev/null +++ b/fluentbit/values.yaml @@ -0,0 +1,256 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for fluentbit +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + fluentbit: docker.io/fluent/fluent-bit:0.14.2 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - fluentbit-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + fluentbit: + template: | + [SERVICE] + Daemon false + Flush 30 + Log_Level info + Parsers_File parsers.conf + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/kern.log + Tag kernel + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Parser docker + Path /var/log/containers/*.log + Tag kube.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/libvirtd.log + Tag libvirt + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/qemu/*.log + Tag qemu + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Tag journal.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=docker.service + Tag journal.* + + [FILTER] + Interval 1s + Match ** + Name throttle + Rate 1000 + Window 300 + + [FILTER] + Match libvirt + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match qemu + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match kernel + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match journal.** + Name modify + Rename _BOOT_ID BOOT_ID + Rename _CAP_EFFECTIVE CAP_EFFECTIVE + Rename _CMDLINE CMDLINE + Rename _COMM COMM + Rename _EXE EXE + Rename _GID GID + Rename _HOSTNAME HOSTNAME + Rename _MACHINE_ID MACHINE_ID + Rename _PID PID + Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP + Rename _SYSTEMD_SLICE SYSTEMD_SLICE + Rename _SYSTEMD_UNIT SYSTEMD_UNIT + Rename _TRANSPORT TRANSPORT + Rename _UID UID + + [OUTPUT] + Match **.fluentd** + Name null + + [FILTER] + Match kube.* + Merge_JSON_Log true + Name kubernetes + + [OUTPUT] + Host ${FLUENTD_HOST} + Match * + Name forward + Port ${FLUENTD_PORT} + parsers: + template: | + [PARSER] + Decode_Field_As escaped_utf8 log + Format json + Name docker + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep true + Time_Key time + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + fluentd: + namespace: null + name: fluentd + hosts: + default: fluentd-logging + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 24224 + metrics: + default: 24220 + +pod: + security_context: + fluentbit: + pod: + runAsUser: 65534 + container: + fluentbit: + runAsUser: 0 + readOnlyRootFilesystem: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + fluentbit: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + fluentbit: + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + tolerations: + fluentbit: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + mounts: + fluentbit: + fluentbit: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_fluentbit: true + job_image_repo_sync: true diff --git a/fluent-logging/Chart.yaml b/fluentd/Chart.yaml similarity index 93% rename from fluent-logging/Chart.yaml rename to fluentd/Chart.yaml index e87238067..a675591ba 100644 --- a/fluent-logging/Chart.yaml +++ b/fluentd/Chart.yaml @@ -14,11 +14,10 @@ apiVersion: v1 description: OpenStack-Helm Fluentd -name: fluent-logging +name: fluentd version: 0.1.0 home: http://www.fluentbit.io/ sources: - - https://github.com/fluent/fluentbit - https://github.com/fluent/fluentd - https://git.openstack.org/cgit/openstack/openstack-helm-infra maintainers: diff --git a/fluentd/requirements.yaml b/fluentd/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/fluentd/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/fluent-logging/templates/bin/_fluentd.sh.tpl b/fluentd/templates/bin/_fluentd.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_fluentd.sh.tpl rename to fluentd/templates/bin/_fluentd.sh.tpl diff --git a/fluentd/templates/configmap-bin.yaml b/fluentd/templates/configmap-bin.yaml new file mode 100644 index 000000000..3ea88b696 --- /dev/null +++ b/fluentd/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-bin +data: + fluentd.sh: | +{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/fluentd/templates/configmap-etc.yaml b/fluentd/templates/configmap-etc.yaml new file mode 100644 index 000000000..ea45589b5 --- /dev/null +++ b/fluentd/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: fluentd-etc +type: Opaque +data: + fluent.conf: {{ .Values.conf.fluentd.template | b64enc }} +{{- end }} diff --git a/fluent-logging/templates/deployment-fluentd.yaml b/fluentd/templates/deployment-fluentd.yaml similarity index 92% rename from fluent-logging/templates/deployment-fluentd.yaml rename to fluentd/templates/deployment-fluentd.yaml index 15fc9241b..f93dd78e1 100644 --- a/fluent-logging/templates/deployment-fluentd.yaml +++ b/fluentd/templates/deployment-fluentd.yaml @@ -95,7 +95,7 @@ spec: configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} spec: -{{ dict "envAll" $envAll "application" "internal" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} +{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} affinity: {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} @@ -108,7 +108,7 @@ spec: - name: fluentd {{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "internal" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/fluentd.sh - start @@ -149,11 +149,11 @@ spec: mountPath: /tmp - name: pod-etc-fluentd mountPath: /fluentd/etc - - name: fluent-logging-etc + - name: fluentd-etc mountPath: /fluentd/etc/fluent.conf subPath: fluent.conf readOnly: true - - name: fluent-logging-bin + - name: fluentd-bin mountPath: /tmp/fluentd.sh subPath: fluentd.sh readOnly: true @@ -163,13 +163,13 @@ spec: emptyDir: {} - name: pod-etc-fluentd emptyDir: {} - - name: fluent-logging-etc + - name: fluentd-etc secret: - secretName: fluent-logging-etc + secretName: fluentd-etc defaultMode: 0444 - - name: fluent-logging-bin + - name: fluentd-bin configMap: - name: fluent-logging-bin + name: fluentd-bin defaultMode: 0555 {{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }} {{- end }} diff --git a/fluentd/templates/job-image-repo-sync.yaml b/fluentd/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..3dd2b0656 --- /dev/null +++ b/fluentd/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl b/fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl rename to fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml b/fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml rename to fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml b/fluentd/templates/monitoring/prometheus/exporter-deployment.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml rename to fluentd/templates/monitoring/prometheus/exporter-deployment.yaml diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-service.yaml b/fluentd/templates/monitoring/prometheus/exporter-service.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-service.yaml rename to fluentd/templates/monitoring/prometheus/exporter-service.yaml diff --git a/fluent-logging/templates/network-policy.yaml b/fluentd/templates/network_policy.yaml similarity index 100% rename from fluent-logging/templates/network-policy.yaml rename to fluentd/templates/network_policy.yaml diff --git a/fluent-logging/templates/secret-elasticsearch-creds.yaml b/fluentd/templates/secret-elasticsearch-creds.yaml similarity index 100% rename from fluent-logging/templates/secret-elasticsearch-creds.yaml rename to fluentd/templates/secret-elasticsearch-creds.yaml diff --git a/fluent-logging/templates/service-fluentd.yaml b/fluentd/templates/service-fluentd.yaml similarity index 100% rename from fluent-logging/templates/service-fluentd.yaml rename to fluentd/templates/service-fluentd.yaml diff --git a/fluent-logging/values.yaml b/fluentd/values.yaml similarity index 78% rename from fluent-logging/values.yaml rename to fluentd/values.yaml index a2f57a785..3d76f7426 100644 --- a/fluent-logging/values.yaml +++ b/fluentd/values.yaml @@ -22,22 +22,12 @@ labels: fluentd: node_selector_key: openstack-control-plane node_selector_value: enabled - fluentbit: - node_selector_key: openstack-control-plane - node_selector_value: enabled prometheus_fluentd_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled - job: - node_selector_key: openstack-control-plane - node_selector_value: enabled - test: - node_selector_key: openstack-control-plane - node_selector_value: enabled images: tags: - fluentbit: docker.io/fluent/fluent-bit:0.14.2 fluentd: docker.io/fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 @@ -60,33 +50,13 @@ dependencies: common: local_image_registry: jobs: - - fluent-logging-image-repo-sync + - fluentd-image-repo-sync services: - endpoint: node service: local_image_registry static: - elasticsearch_template: - services: - - endpoint: internal - service: elasticsearch - fluentbit: - jobs: - - elasticsearch-template - services: - - endpoint: internal - service: fluentd fluentd: - jobs: - - elasticsearch-template - services: - - endpoint: internal - service: elasticsearch - fluentd_with_kafka: - services: - - endpoint: internal - service: elasticsearch - - endpoint: public - service: kafka + services: null image_repo_sync: services: - endpoint: internal @@ -95,12 +65,6 @@ dependencies: services: - endpoint: internal service: fluentd - tests: - services: - - endpoint: internal - service: elasticsearch - - endpoint: internal - service: fluentd conf: fluentd: @@ -436,40 +400,10 @@ conf: Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Keep true Time_Key time - fluentd_exporter: log: format: "logger:stdout?json=true" level: "info" - templates: - fluent: - template: "logstash-*" - index_patterns: "logstash-*" - settings: - number_of_shards: 1 - mappings: - fluent: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false endpoints: cluster_domain_suffix: cluster.local @@ -506,21 +440,6 @@ endpoints: port: http: default: 80 - kafka: - namespace: null - name: kafka - hosts: - default: kafka-logging - public: kafka - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - service: - default: 9092 fluentd: namespace: null name: fluentd @@ -577,34 +496,13 @@ network_policy: pod: security_context: - daemon: - pod: - runAsUser: 65534 - container: - fluentbit: - runAsUser: 0 - readOnlyRootFilesystem: true - internal: + fluentd: pod: runAsUser: 65534 container: fluentd: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - elasticsearch_template: - pod: - runAsUser: 65534 - container: - elasticsearch_template: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - test: - pod: - runAsUser: 65534 - container: - helm_test: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true exporter: pod: runAsUser: 65534 @@ -622,12 +520,6 @@ pod: default: 10 lifecycle: upgrades: - daemonsets: - pod_replacement_strategy: RollingUpdate - fluentbit: - enabled: true - min_ready_seconds: 0 - max_unavailable: 1 deployments: revision_history: 3 pod_replacement_strategy: RollingUpdate @@ -644,13 +536,6 @@ pod: prometheus_fluentd_exporter: 1 resources: enabled: false - fluentbit: - limits: - memory: '400Mi' - cpu: '400m' - requests: - memory: '100Mi' - cpu: '100m' fluentd: limits: memory: '1024Mi' @@ -665,47 +550,15 @@ pod: requests: memory: "128Mi" cpu: "500m" - jobs: - image_repo_sync: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - tests: - requests: - memory: '128Mi' - cpu: '100m' - limits: - memory: '1024Mi' - cpu: '2000m' - tolerations: - fluentbit: - enabled: false - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - - key: node-role.kubernetes.io/node - operator: Exists mounts: fluentd: fluentd: - fluentbit: - fluentbit: - fluent_tests: - fluent_tests: - elasticsearch_template: - init_container: - elasticsearch_template: manifests: configmap_bin: true configmap_etc: true deployment_fluentd: true - daemonset_fluentbit: true job_image_repo_sync: true - helm_tests: true monitoring: prometheus: configmap_bin_exporter: true @@ -715,4 +568,3 @@ manifests: network_policy: false secret_elasticsearch: true service_fluentd: true - job_elasticsearch_template: true diff --git a/tools/deployment/armada/manifests/armada-lma.yaml b/tools/deployment/armada/manifests/armada-lma.yaml index 968dff685..bc8f45c1d 100644 --- a/tools/deployment/armada/manifests/armada-lma.yaml +++ b/tools/deployment/armada/manifests/armada-lma.yaml @@ -328,6 +328,81 @@ data: master: "-Xms512m -Xmx512m" snapshots: enabled: true + templates: + syslog: + template: "syslog-*" + index_patterns: "syslog-*" + settings: + number_of_shards: 1 + mappings: + syslog: + properties: + cluster: + type: keyword + app: + type: keyword + pid: + type: integer + host: + type: keyword + log: + type: text + oslo_openstack_fluentd: + template: "openstack-*" + index_patterns: "openstack-*" + settings: + number_of_shards: 1 + mappings: + oslo_openstack_fluentd: + properties: + extra: + properties: + project: + type: text + norms: false + version: + type: text + norms: false + filename: + type: text + norms: false + funcname: + type: text + norms: false + message: + type: text + norms: false + process_name: + type: keyword + index: false + docker_fluentd: + template: "logstash-*" + index_patterns: "logstash-*" + settings: + number_of_shards: 1 + mappings: + docker_fluentd: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false source: type: local location: ${OSH_INFRA_PATH} @@ -339,19 +414,17 @@ data: schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 - name: fluent-logging + name: fluentbit data: - chart_name: fluent-logging - release: fluent-logging + chart_name: fluentbit + release: fluentbit namespace: osh-infra wait: timeout: 3600 labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit resources: - type: daemonset - - type: deployment - - type: job test: timeout: 600 install: @@ -362,10 +435,52 @@ data: delete: - type: job labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit - type: pod labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit + component: test + values: + release_uuid: ${RELEASE_UUID} + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + source: + type: local + location: ${OSH_INFRA_PATH} + subpath: fluentbit + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: fluentd +data: + chart_name: fluentd + release: fluentd + namespace: osh-infra + wait: + timeout: 3600 + labels: + release_group: osh-infra-fluentd + resources: + - type: deployment + test: + timeout: 600 + install: + no_hooks: False + upgrade: + no_hooks: False + pre: + delete: + - type: job + labels: + release_group: osh-infra-fluentd + - type: pod + labels: + release_group: osh-infra-fluentd component: test values: release_uuid: ${RELEASE_UUID} @@ -384,9 +499,6 @@ data: fluentd: node_selector_key: openstack-control-plane node_selector_value: enabled - fluentbit: - node_selector_key: openstack-control-plane - node_selector_value: enabled prometheus_fluentd_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled @@ -396,7 +508,7 @@ data: source: type: local location: ${OSH_INFRA_PATH} - subpath: fluent-logging + subpath: fluentd reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/common/fluentbit.sh b/tools/deployment/common/fluentbit.sh new file mode 100755 index 000000000..93c106878 --- /dev/null +++ b/tools/deployment/common/fluentbit.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make fluentbit + +helm upgrade --install fluentbit ./fluentbit \ + --namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status fluentbit diff --git a/tools/deployment/multinode/125-fluentbit.sh b/tools/deployment/multinode/125-fluentbit.sh new file mode 120000 index 000000000..0ed92806a --- /dev/null +++ b/tools/deployment/multinode/125-fluentbit.sh @@ -0,0 +1 @@ +../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/multinode/130-fluent-logging.sh b/tools/deployment/multinode/130-fluentd.sh similarity index 75% rename from tools/deployment/multinode/130-fluent-logging.sh rename to tools/deployment/multinode/130-fluentd.sh index 50bfe63e8..c1af05064 100755 --- a/tools/deployment/multinode/130-fluent-logging.sh +++ b/tools/deployment/multinode/130-fluentd.sh @@ -17,10 +17,10 @@ set -xe #NOTE: Lint and package chart -make fluent-logging +make fluentd if [ ! -d "/var/log/journal" ]; then -tee /tmp/fluent-logging.yaml << EOF +tee /tmp/fluentd.yaml << EOF monitoring: prometheus: enabled: true @@ -36,20 +36,16 @@ pod: - name: runlog mountPath: /run/log EOF -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ - --values=/tmp/fluent-logging.yaml + --values=/tmp/fluentd.yaml else -helm upgrade --install fluent-logging ./fluent-logging \ - --namespace=osh-infra \ - --set monitoring.prometheus.enabled=true +helm upgrade --install fluentd ./fluentd \ + --namespace=osh-infra fi #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh osh-infra #NOTE: Validate Deployment info -helm status fluent-logging - -#NOTE: Run helm tests -helm test fluent-logging +helm status fluentd diff --git a/tools/deployment/osh-infra-logging/060-fluentbit.sh b/tools/deployment/osh-infra-logging/060-fluentbit.sh new file mode 120000 index 000000000..0ed92806a --- /dev/null +++ b/tools/deployment/osh-infra-logging/060-fluentbit.sh @@ -0,0 +1 @@ +../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/060-fluent-logging.sh b/tools/deployment/osh-infra-logging/065-fluentd.sh similarity index 82% rename from tools/deployment/osh-infra-logging/060-fluent-logging.sh rename to tools/deployment/osh-infra-logging/065-fluentd.sh index ea95267df..e911d452a 100755 --- a/tools/deployment/osh-infra-logging/060-fluent-logging.sh +++ b/tools/deployment/osh-infra-logging/065-fluentd.sh @@ -17,10 +17,10 @@ set -xe #NOTE: Lint and package chart -make fluent-logging +make fluentd if [ ! -d "/var/log/journal" ]; then -tee /tmp/fluent-logging.yaml << EOF +tee /tmp/fluentd.yaml << EOF monitoring: prometheus: enabled: true @@ -38,11 +38,11 @@ pod: - name: runlog mountPath: /run/log EOF -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ - --values=/tmp/fluent-logging.yaml + --values=/tmp/fluentd.yaml else -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ --set pod.replicas.fluentd=1 \ --set monitoring.prometheus.enabled=true @@ -52,6 +52,4 @@ fi ./tools/deployment/common/wait-for-pods.sh osh-infra #NOTE: Validate Deployment info -helm status fluent-logging - -helm test fluent-logging +helm status fluentd diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8d1cfe40e..e222c59ce 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -60,7 +60,8 @@ - ./tools/deployment/multinode/110-nagios.sh - ./tools/deployment/multinode/115-radosgw-osh-infra.sh - ./tools/deployment/multinode/120-elasticsearch.sh - - ./tools/deployment/multinode/130-fluent-logging.sh + - ./tools/deployment/multinode/125-fluentbit.sh + - ./tools/deployment/multinode/130-fluentd.sh - ./tools/deployment/multinode/140-kibana.sh - ./tools/deployment/multinode/600-grafana-selenium.sh - ./tools/deployment/multinode/610-nagios-selenium.sh @@ -127,7 +128,8 @@ - ./tools/deployment/osh-infra-logging/040-ldap.sh - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh - ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh - - ./tools/deployment/osh-infra-logging/060-fluent-logging.sh + - ./tools/deployment/osh-infra-logging/060-fluentbit.sh + - ./tools/deployment/osh-infra-logging/065-fluentd.sh - ./tools/deployment/osh-infra-logging/070-kibana.sh - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh