diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index 9246af02b..d06f002e4 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -210,10 +210,10 @@ Alternatively, this step can be performed by running the script directly: ./tools/deployment/multinode/120-elasticsearch.sh -Deploy Fluent-Logging -^^^^^^^^^^^^^^^^^^^^^ +Deploy Fluentbit +^^^^^^^^^^^^^^^^ -.. literalinclude:: ../../../tools/deployment/multinode/130-fluent-logging.sh +.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh :language: shell :lines: 1,17- @@ -221,4 +221,17 @@ Alternatively, this step can be performed by running the script directly: .. code-block:: shell - ./tools/deployment/multinode/130-fluent-logging.sh + ./tools/deployment/multinode/125-fluentbit.sh + +Deploy Fluentd +^^^^^^^^^^^^^^ + +.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh + :language: shell + :lines: 1,17- + +Alternatively, this step can be performed by running the script directly: + +.. code-block:: shell + + ./tools/deployment/multinode/130-fluentd.sh diff --git a/fluent-logging/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_create_template.sh.tpl rename to elasticsearch/templates/bin/_create_template.sh.tpl diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index 80c02a830..877659232 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -52,6 +52,24 @@ function check_snapshot_repositories () { {{ end }} } +{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} +# Tests whether elasticsearch has successfully generated the elasticsearch index mapping +# templates defined by values.yaml +function check_templates () { + {{ range $template, $fields := .Values.conf.templates }} + {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ + | python -c "import sys, json; print len(json.load(sys.stdin))") + if [ "${{$template}}_total_hits" -gt 0 ]; then + echo "PASS: Successful hits on {{$template}} template!" + else + echo "FAIL: No hits on query for {{$template}} template! Exiting"; + exit 1; + fi + {{ end }} +} +{{ end }} + function remove_test_index () { echo "Deleting index created for service testing" curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ @@ -63,4 +81,5 @@ create_test_index {{ if .Values.conf.elasticsearch.snapshots.enabled }} check_snapshot_repositories {{ end }} +check_templates remove_test_index diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml index 4f213835f..6627e2d2d 100644 --- a/elasticsearch/templates/configmap-bin-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -38,6 +38,8 @@ data: {{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} es-cluster-wait.sh: | {{ tuple "bin/_es-cluster-wait.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_template.sh: | +{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml b/elasticsearch/templates/configmap-etc-templates.yaml similarity index 54% rename from fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml rename to elasticsearch/templates/configmap-etc-templates.yaml index b101d9fd5..5e2ea5732 100644 --- a/fluent-logging/templates/monitoring/prometheus/exporter-network-policy.yaml +++ b/elasticsearch/templates/configmap-etc-templates.yaml @@ -1,11 +1,11 @@ {{/* -Copyright 2017-2018 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} -{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-fluentd-exporter" }} -{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} -{{- end -}} +{{- if .Values.manifests.configmap_etc_templates }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-templates-etc +type: Opaque +data: +{{ range $template, $fields := .Values.conf.templates }} + {{ $template }}.json: {{ toJson $fields | b64enc }} +{{ end }} +{{- end }} diff --git a/fluent-logging/templates/job-elasticsearch-template.yaml b/elasticsearch/templates/job-elasticsearch-template.yaml similarity index 66% rename from fluent-logging/templates/job-elasticsearch-template.yaml rename to elasticsearch/templates/job-elasticsearch-template.yaml index 5b18df715..d563d7ee1 100644 --- a/fluent-logging/templates/job-elasticsearch-template.yaml +++ b/elasticsearch/templates/job-elasticsearch-template.yaml @@ -14,26 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} +{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} -{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }} -{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }} +{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }} +{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }} -{{- $serviceAccountName := "fluent-logging-elasticsearch-template" }} -{{ tuple $envAll "elasticsearch_template" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- $serviceAccountName := "create-elasticsearch-templates" }} +{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job metadata: - name: elasticsearch-template + name: create-elasticsearch-templates annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} spec: template: metadata: labels: -{{ tuple $envAll "fluentd" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: {{ dict "envAll" $envAll "application" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} @@ -41,11 +41,11 @@ spec: nodeSelector: {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} initContainers: -{{ tuple $envAll "elasticsearch_template" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - - name: elasticsearch-template -{{ tuple $envAll "elasticsearch_template" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + - name: create-elasticsearch-templates +{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "elasticsearch_template" "container" "elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} env: - name: ELASTICSEARCH_HOST @@ -67,27 +67,27 @@ spec: volumeMounts: - name: pod-tmp mountPath: /tmp - - name: fluent-logging-bin + - name: elasticsearch-bin mountPath: /tmp/create_template.sh subPath: create_template.sh readOnly: true {{ range $template, $fields := .Values.conf.templates }} - - name: fluent-logging-etc + - name: elasticsearch-templates-etc mountPath: /tmp/{{$template}}.json subPath: {{$template}}.json readOnly: true {{ end }} -{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 12 }}{{ end }} +{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }} volumes: - name: pod-tmp emptyDir: {} - - name: fluent-logging-bin + - name: elasticsearch-bin configMap: - name: fluent-logging-bin + name: elasticsearch-bin defaultMode: 0555 - - name: fluent-logging-etc + - name: elasticsearch-templates-etc secret: - secretName: fluent-logging-etc + secretName: elasticsearch-templates-etc defaultMode: 0444 -{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 8 }}{{ end }} +{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 57f397f14..1fe09a089 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -30,6 +30,7 @@ images: dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 snapshot_repository: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial es_cluster_wait: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_xenial + elasticsearch_templates: docker.io/openstackhelm/heat:newton image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" local_registry: @@ -82,6 +83,10 @@ dependencies: services: - endpoint: internal service: elasticsearch + elasticsearch_templates: + services: null + jobs: + - elasticsearch-cluster-wait image_repo_sync: services: - endpoint: internal @@ -220,6 +225,8 @@ pod: mounts: elasticsearch: elasticsearch: + elasticsearch_templates: + elasticsearch_templates: resources: enabled: false apache_proxy: @@ -265,6 +272,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + elasticsearch_templates: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" image_repo_sync: requests: memory: "128Mi" @@ -635,6 +649,35 @@ conf: es: all: true timeout: 20s + templates: + fluent: + template: "logstash-*" + index_patterns: "logstash-*" + settings: + number_of_shards: 1 + mappings: + fluent: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false endpoints: cluster_domain_suffix: cluster.local @@ -771,11 +814,13 @@ manifests: configmap_bin_elasticsearch: true configmap_etc_curator: true configmap_etc_elasticsearch: true + configmap_etc_templates: true cron_curator: true deployment_client: true deployment_master: true ingress: true job_cluster_wait: true + job_elasticsearch_templates: true job_image_repo_sync: true job_snapshot_repository: true job_s3_user: true diff --git a/fluent-logging/README.rst b/fluent-logging/README.rst deleted file mode 100644 index c2f035db4..000000000 --- a/fluent-logging/README.rst +++ /dev/null @@ -1,30 +0,0 @@ -Fluentd-logging -=============== - -OpenStack-Helm defines a centralized logging mechanism to provide insight into -the state of the OpenStack services and infrastructure components as -well as underlying kubernetes platform. Among the requirements for a logging -platform, where log data can come from and where log data need to be delivered -are very variable. To support various logging scenarios, OpenStack-Helm should -provide a flexible mechanism to meet with certain operation needs. This chart -proposes fast and lightweight log forwarder and full featured log aggregator -complementing each other providing a flexible and reliable solution. Especially, -Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log -aggregator and processor. - - -Mechanism ---------- - -Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering, -aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on -each node and mounts the /var/lib/docker/containers directory. The Docker -container runtime engine directs events posted to stdout and stderr to this -directory on the host. Fluent-bit then forward the contents of that directory to -Fluentd. Fluentd runs as deployment at the designated nodes and expose service -for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to -the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs. -Fluentd will then forward the results to Elasticsearch and to optionally kafka. -Elasticsearch indexes the logs in a logstash-* index by default. kafka stores -the logs in a 'logs' topic by default. Any external tool can then consume the -'logs' topic. diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl deleted file mode 100644 index 24d94f11a..000000000 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex - -{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} -# Tests whether fluent-logging has successfully generated the elasticsearch index mapping -# templates defined by values.yaml -function check_templates () { - {{ range $template, $fields := .Values.conf.templates }} - {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ - | python -c "import sys, json; print len(json.load(sys.stdin))") - if [ "${{$template}}_total_hits" -gt 0 ]; then - echo "PASS: Successful hits on {{$template}} template, provided by fluent-logging!" - else - echo "FAIL: No hits on query for {{$template}} template! Exiting"; - exit 1; - fi - {{ end }} -} -{{ end }} - -{{ if and (.Values.manifests.job_elasticsearch_template) (not (empty .Values.conf.templates)) }} -check_templates -{{ end }} diff --git a/fluent-logging/templates/pod-helm-tests.yaml b/fluent-logging/templates/pod-helm-tests.yaml deleted file mode 100644 index 053e66a4e..000000000 --- a/fluent-logging/templates/pod-helm-tests.yaml +++ /dev/null @@ -1,75 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.helm_tests }} -{{- $envAll := . }} -{{- $esUserSecret := .Values.secrets.elasticsearch.user }} - -{{- $serviceAccountName := print .Release.Name "-test" }} -{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: v1 -kind: Pod -metadata: - name: "{{.Release.Name}}-test" - labels: -{{ tuple $envAll "fluentd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - annotations: - "helm.sh/hook": test-success - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: -{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} - restartPolicy: Never - initContainers: -{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} - containers: - - name: {{.Release.Name}}-helm-tests -{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - command: - - /tmp/helm-tests.sh - env: - - name: ELASTICSEARCH_USERNAME - valueFrom: - secretKeyRef: - name: {{ $esUserSecret }} - key: ELASTICSEARCH_USERNAME - - name: ELASTICSEARCH_PASSWORD - valueFrom: - secretKeyRef: - name: {{ $esUserSecret }} - key: ELASTICSEARCH_PASSWORD - - name: ELASTICSEARCH_ENDPOINT - value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} - volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: fluent-logging-bin - mountPath: /tmp/helm-tests.sh - subPath: helm-tests.sh - readOnly: true - volumes: - - name: pod-tmp - emptyDir: {} - - name: fluent-logging-bin - configMap: - name: fluent-logging-bin - defaultMode: 0555 -{{- end }} diff --git a/fluentbit/Chart.yaml b/fluentbit/Chart.yaml new file mode 100644 index 000000000..7a88e5e0d --- /dev/null +++ b/fluentbit/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Fluentbit +name: fluentbit +version: 0.1.0 +home: http://www.fluentbit.io/ +sources: + - https://github.com/fluent/fluentbit + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/fluent-logging/requirements.yaml b/fluentbit/requirements.yaml similarity index 100% rename from fluent-logging/requirements.yaml rename to fluentbit/requirements.yaml diff --git a/fluent-logging/templates/bin/_fluent-bit.sh.tpl b/fluentbit/templates/bin/_fluent-bit.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_fluent-bit.sh.tpl rename to fluentbit/templates/bin/_fluent-bit.sh.tpl diff --git a/fluent-logging/templates/configmap-bin.yaml b/fluentbit/templates/configmap-bin.yaml similarity index 71% rename from fluent-logging/templates/configmap-bin.yaml rename to fluentbit/templates/configmap-bin.yaml index e331e36e1..ea63b42f9 100644 --- a/fluent-logging/templates/configmap-bin.yaml +++ b/fluentbit/templates/configmap-bin.yaml @@ -20,16 +20,10 @@ limitations under the License. apiVersion: v1 kind: ConfigMap metadata: - name: fluent-logging-bin + name: fluentbit-bin data: - fluentd.sh: | -{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} fluent-bit.sh: | {{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - helm-tests.sh: | -{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - create_template.sh: | -{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluentbit/templates/configmap-etc.yaml similarity index 80% rename from fluent-logging/templates/configmap-etc.yaml rename to fluentbit/templates/configmap-etc.yaml index fae65dcac..dcafb1eec 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluentbit/templates/configmap-etc.yaml @@ -20,13 +20,9 @@ limitations under the License. apiVersion: v1 kind: Secret metadata: - name: fluent-logging-etc + name: fluentbit-etc type: Opaque data: fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }} parsers.conf: {{ .Values.conf.parsers.template | b64enc }} - fluent.conf: {{ .Values.conf.fluentd.template | b64enc }} -{{ range $template, $fields := .Values.conf.templates }} - {{ $template }}.json: {{ toJson $fields | b64enc }} -{{ end }} {{- end }} diff --git a/fluent-logging/templates/daemonset-fluent-bit.yaml b/fluentbit/templates/daemonset-fluent-bit.yaml similarity index 90% rename from fluent-logging/templates/daemonset-fluent-bit.yaml rename to fluentbit/templates/daemonset-fluent-bit.yaml index abe9b841f..f9c59e371 100644 --- a/fluent-logging/templates/daemonset-fluent-bit.yaml +++ b/fluentbit/templates/daemonset-fluent-bit.yaml @@ -95,7 +95,7 @@ spec: configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} {{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} spec: -{{ dict "envAll" $envAll "application" "daemon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} +{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} {{ if $envAll.Values.pod.tolerations.fluentbit.enabled }} {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} @@ -112,7 +112,7 @@ spec: - name: fluentbit {{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "daemon" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/fluent-bit.sh env: @@ -123,7 +123,7 @@ spec: volumeMounts: - name: pod-tmp mountPath: /tmp - - name: fluent-logging-bin + - name: fluentbit-bin mountPath: /tmp/fluent-bit.sh subPath: fluent-bit.sh readOnly: true @@ -133,11 +133,11 @@ spec: - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - - name: fluent-logging-etc + - name: fluentbit-etc mountPath: /fluent-bit/etc/fluent-bit.conf subPath: fluent-bit.conf readOnly: true - - name: fluent-logging-etc + - name: fluentbit-etc mountPath: /fluent-bit/etc/parsers.conf subPath: parsers.conf readOnly: true @@ -151,13 +151,13 @@ spec: - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - - name: fluent-logging-bin + - name: fluentbit-bin configMap: - name: fluent-logging-bin + name: fluentbit-bin defaultMode: 0555 - - name: fluent-logging-etc + - name: fluentbit-etc secret: - secretName: fluent-logging-etc + secretName: fluentbit-etc defaultMode: 0444 {{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/fluent-logging/templates/job-image-repo-sync.yaml b/fluentbit/templates/job-image-repo-sync.yaml similarity index 90% rename from fluent-logging/templates/job-image-repo-sync.yaml rename to fluentbit/templates/job-image-repo-sync.yaml index 02c56ab7e..0916c7107 100644 --- a/fluent-logging/templates/job-image-repo-sync.yaml +++ b/fluentbit/templates/job-image-repo-sync.yaml @@ -15,6 +15,6 @@ limitations under the License. */}} {{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} -{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluent-logging" -}} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}} {{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} {{- end }} diff --git a/fluentbit/values.yaml b/fluentbit/values.yaml new file mode 100644 index 000000000..ac30a603b --- /dev/null +++ b/fluentbit/values.yaml @@ -0,0 +1,256 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for fluentbit +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + fluentbit: docker.io/fluent/fluent-bit:0.14.2 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - fluentbit-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + fluentbit: + template: | + [SERVICE] + Daemon false + Flush 30 + Log_Level info + Parsers_File parsers.conf + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/kern.log + Tag kernel + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Parser docker + Path /var/log/containers/*.log + Tag kube.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/libvirtd.log + Tag libvirt + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/qemu/*.log + Tag qemu + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Tag journal.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=docker.service + Tag journal.* + + [FILTER] + Interval 1s + Match ** + Name throttle + Rate 1000 + Window 300 + + [FILTER] + Match libvirt + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match qemu + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match kernel + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match journal.** + Name modify + Rename _BOOT_ID BOOT_ID + Rename _CAP_EFFECTIVE CAP_EFFECTIVE + Rename _CMDLINE CMDLINE + Rename _COMM COMM + Rename _EXE EXE + Rename _GID GID + Rename _HOSTNAME HOSTNAME + Rename _MACHINE_ID MACHINE_ID + Rename _PID PID + Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP + Rename _SYSTEMD_SLICE SYSTEMD_SLICE + Rename _SYSTEMD_UNIT SYSTEMD_UNIT + Rename _TRANSPORT TRANSPORT + Rename _UID UID + + [OUTPUT] + Match **.fluentd** + Name null + + [FILTER] + Match kube.* + Merge_JSON_Log true + Name kubernetes + + [OUTPUT] + Host ${FLUENTD_HOST} + Match * + Name forward + Port ${FLUENTD_PORT} + parsers: + template: | + [PARSER] + Decode_Field_As escaped_utf8 log + Format json + Name docker + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep true + Time_Key time + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + fluentd: + namespace: null + name: fluentd + hosts: + default: fluentd-logging + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 24224 + metrics: + default: 24220 + +pod: + security_context: + fluentbit: + pod: + runAsUser: 65534 + container: + fluentbit: + runAsUser: 0 + readOnlyRootFilesystem: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + fluentbit: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + fluentbit: + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + tolerations: + fluentbit: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + mounts: + fluentbit: + fluentbit: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_fluentbit: true + job_image_repo_sync: true diff --git a/fluent-logging/Chart.yaml b/fluentd/Chart.yaml similarity index 93% rename from fluent-logging/Chart.yaml rename to fluentd/Chart.yaml index e87238067..a675591ba 100644 --- a/fluent-logging/Chart.yaml +++ b/fluentd/Chart.yaml @@ -14,11 +14,10 @@ apiVersion: v1 description: OpenStack-Helm Fluentd -name: fluent-logging +name: fluentd version: 0.1.0 home: http://www.fluentbit.io/ sources: - - https://github.com/fluent/fluentbit - https://github.com/fluent/fluentd - https://git.openstack.org/cgit/openstack/openstack-helm-infra maintainers: diff --git a/fluentd/requirements.yaml b/fluentd/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/fluentd/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/fluent-logging/templates/bin/_fluentd.sh.tpl b/fluentd/templates/bin/_fluentd.sh.tpl similarity index 100% rename from fluent-logging/templates/bin/_fluentd.sh.tpl rename to fluentd/templates/bin/_fluentd.sh.tpl diff --git a/fluentd/templates/configmap-bin.yaml b/fluentd/templates/configmap-bin.yaml new file mode 100644 index 000000000..3ea88b696 --- /dev/null +++ b/fluentd/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-bin +data: + fluentd.sh: | +{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/fluentd/templates/configmap-etc.yaml b/fluentd/templates/configmap-etc.yaml new file mode 100644 index 000000000..ea45589b5 --- /dev/null +++ b/fluentd/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: fluentd-etc +type: Opaque +data: + fluent.conf: {{ .Values.conf.fluentd.template | b64enc }} +{{- end }} diff --git a/fluent-logging/templates/deployment-fluentd.yaml b/fluentd/templates/deployment-fluentd.yaml similarity index 92% rename from fluent-logging/templates/deployment-fluentd.yaml rename to fluentd/templates/deployment-fluentd.yaml index 15fc9241b..f93dd78e1 100644 --- a/fluent-logging/templates/deployment-fluentd.yaml +++ b/fluentd/templates/deployment-fluentd.yaml @@ -95,7 +95,7 @@ spec: configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} spec: -{{ dict "envAll" $envAll "application" "internal" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} +{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} serviceAccountName: {{ $serviceAccountName }} affinity: {{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} @@ -108,7 +108,7 @@ spec: - name: fluentd {{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "internal" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} command: - /tmp/fluentd.sh - start @@ -149,11 +149,11 @@ spec: mountPath: /tmp - name: pod-etc-fluentd mountPath: /fluentd/etc - - name: fluent-logging-etc + - name: fluentd-etc mountPath: /fluentd/etc/fluent.conf subPath: fluent.conf readOnly: true - - name: fluent-logging-bin + - name: fluentd-bin mountPath: /tmp/fluentd.sh subPath: fluentd.sh readOnly: true @@ -163,13 +163,13 @@ spec: emptyDir: {} - name: pod-etc-fluentd emptyDir: {} - - name: fluent-logging-etc + - name: fluentd-etc secret: - secretName: fluent-logging-etc + secretName: fluentd-etc defaultMode: 0444 - - name: fluent-logging-bin + - name: fluentd-bin configMap: - name: fluent-logging-bin + name: fluentd-bin defaultMode: 0555 {{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }} {{- end }} diff --git a/fluentd/templates/job-image-repo-sync.yaml b/fluentd/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..3dd2b0656 --- /dev/null +++ b/fluentd/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl b/fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl rename to fluentd/templates/monitoring/prometheus/bin/_fluentd-exporter.sh.tpl diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml b/fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-configmap-bin.yaml rename to fluentd/templates/monitoring/prometheus/exporter-configmap-bin.yaml diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml b/fluentd/templates/monitoring/prometheus/exporter-deployment.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-deployment.yaml rename to fluentd/templates/monitoring/prometheus/exporter-deployment.yaml diff --git a/fluent-logging/templates/monitoring/prometheus/exporter-service.yaml b/fluentd/templates/monitoring/prometheus/exporter-service.yaml similarity index 100% rename from fluent-logging/templates/monitoring/prometheus/exporter-service.yaml rename to fluentd/templates/monitoring/prometheus/exporter-service.yaml diff --git a/fluent-logging/templates/network-policy.yaml b/fluentd/templates/network_policy.yaml similarity index 100% rename from fluent-logging/templates/network-policy.yaml rename to fluentd/templates/network_policy.yaml diff --git a/fluent-logging/templates/secret-elasticsearch-creds.yaml b/fluentd/templates/secret-elasticsearch-creds.yaml similarity index 100% rename from fluent-logging/templates/secret-elasticsearch-creds.yaml rename to fluentd/templates/secret-elasticsearch-creds.yaml diff --git a/fluent-logging/templates/service-fluentd.yaml b/fluentd/templates/service-fluentd.yaml similarity index 100% rename from fluent-logging/templates/service-fluentd.yaml rename to fluentd/templates/service-fluentd.yaml diff --git a/fluent-logging/values.yaml b/fluentd/values.yaml similarity index 78% rename from fluent-logging/values.yaml rename to fluentd/values.yaml index a2f57a785..3d76f7426 100644 --- a/fluent-logging/values.yaml +++ b/fluentd/values.yaml @@ -22,22 +22,12 @@ labels: fluentd: node_selector_key: openstack-control-plane node_selector_value: enabled - fluentbit: - node_selector_key: openstack-control-plane - node_selector_value: enabled prometheus_fluentd_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled - job: - node_selector_key: openstack-control-plane - node_selector_value: enabled - test: - node_selector_key: openstack-control-plane - node_selector_value: enabled images: tags: - fluentbit: docker.io/fluent/fluent-bit:0.14.2 fluentd: docker.io/fluent/fluentd-kubernetes-daemonset:v1.2-debian-elasticsearch prometheus_fluentd_exporter: docker.io/bitnami/fluentd-exporter:0.2.0 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 @@ -60,33 +50,13 @@ dependencies: common: local_image_registry: jobs: - - fluent-logging-image-repo-sync + - fluentd-image-repo-sync services: - endpoint: node service: local_image_registry static: - elasticsearch_template: - services: - - endpoint: internal - service: elasticsearch - fluentbit: - jobs: - - elasticsearch-template - services: - - endpoint: internal - service: fluentd fluentd: - jobs: - - elasticsearch-template - services: - - endpoint: internal - service: elasticsearch - fluentd_with_kafka: - services: - - endpoint: internal - service: elasticsearch - - endpoint: public - service: kafka + services: null image_repo_sync: services: - endpoint: internal @@ -95,12 +65,6 @@ dependencies: services: - endpoint: internal service: fluentd - tests: - services: - - endpoint: internal - service: elasticsearch - - endpoint: internal - service: fluentd conf: fluentd: @@ -436,40 +400,10 @@ conf: Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Keep true Time_Key time - fluentd_exporter: log: format: "logger:stdout?json=true" level: "info" - templates: - fluent: - template: "logstash-*" - index_patterns: "logstash-*" - settings: - number_of_shards: 1 - mappings: - fluent: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false endpoints: cluster_domain_suffix: cluster.local @@ -506,21 +440,6 @@ endpoints: port: http: default: 80 - kafka: - namespace: null - name: kafka - hosts: - default: kafka-logging - public: kafka - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - service: - default: 9092 fluentd: namespace: null name: fluentd @@ -577,34 +496,13 @@ network_policy: pod: security_context: - daemon: - pod: - runAsUser: 65534 - container: - fluentbit: - runAsUser: 0 - readOnlyRootFilesystem: true - internal: + fluentd: pod: runAsUser: 65534 container: fluentd: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - elasticsearch_template: - pod: - runAsUser: 65534 - container: - elasticsearch_template: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - test: - pod: - runAsUser: 65534 - container: - helm_test: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true exporter: pod: runAsUser: 65534 @@ -622,12 +520,6 @@ pod: default: 10 lifecycle: upgrades: - daemonsets: - pod_replacement_strategy: RollingUpdate - fluentbit: - enabled: true - min_ready_seconds: 0 - max_unavailable: 1 deployments: revision_history: 3 pod_replacement_strategy: RollingUpdate @@ -644,13 +536,6 @@ pod: prometheus_fluentd_exporter: 1 resources: enabled: false - fluentbit: - limits: - memory: '400Mi' - cpu: '400m' - requests: - memory: '100Mi' - cpu: '100m' fluentd: limits: memory: '1024Mi' @@ -665,47 +550,15 @@ pod: requests: memory: "128Mi" cpu: "500m" - jobs: - image_repo_sync: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - tests: - requests: - memory: '128Mi' - cpu: '100m' - limits: - memory: '1024Mi' - cpu: '2000m' - tolerations: - fluentbit: - enabled: false - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - - key: node-role.kubernetes.io/node - operator: Exists mounts: fluentd: fluentd: - fluentbit: - fluentbit: - fluent_tests: - fluent_tests: - elasticsearch_template: - init_container: - elasticsearch_template: manifests: configmap_bin: true configmap_etc: true deployment_fluentd: true - daemonset_fluentbit: true job_image_repo_sync: true - helm_tests: true monitoring: prometheus: configmap_bin_exporter: true @@ -715,4 +568,3 @@ manifests: network_policy: false secret_elasticsearch: true service_fluentd: true - job_elasticsearch_template: true diff --git a/tools/deployment/armada/manifests/armada-lma.yaml b/tools/deployment/armada/manifests/armada-lma.yaml index 968dff685..bc8f45c1d 100644 --- a/tools/deployment/armada/manifests/armada-lma.yaml +++ b/tools/deployment/armada/manifests/armada-lma.yaml @@ -328,6 +328,81 @@ data: master: "-Xms512m -Xmx512m" snapshots: enabled: true + templates: + syslog: + template: "syslog-*" + index_patterns: "syslog-*" + settings: + number_of_shards: 1 + mappings: + syslog: + properties: + cluster: + type: keyword + app: + type: keyword + pid: + type: integer + host: + type: keyword + log: + type: text + oslo_openstack_fluentd: + template: "openstack-*" + index_patterns: "openstack-*" + settings: + number_of_shards: 1 + mappings: + oslo_openstack_fluentd: + properties: + extra: + properties: + project: + type: text + norms: false + version: + type: text + norms: false + filename: + type: text + norms: false + funcname: + type: text + norms: false + message: + type: text + norms: false + process_name: + type: keyword + index: false + docker_fluentd: + template: "logstash-*" + index_patterns: "logstash-*" + settings: + number_of_shards: 1 + mappings: + docker_fluentd: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false source: type: local location: ${OSH_INFRA_PATH} @@ -339,19 +414,17 @@ data: schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 - name: fluent-logging + name: fluentbit data: - chart_name: fluent-logging - release: fluent-logging + chart_name: fluentbit + release: fluentbit namespace: osh-infra wait: timeout: 3600 labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit resources: - type: daemonset - - type: deployment - - type: job test: timeout: 600 install: @@ -362,10 +435,52 @@ data: delete: - type: job labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit - type: pod labels: - release_group: osh-infra-fluent-logging + release_group: osh-infra-fluentbit + component: test + values: + release_uuid: ${RELEASE_UUID} + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + source: + type: local + location: ${OSH_INFRA_PATH} + subpath: fluentbit + reference: master + dependencies: + - helm-toolkit +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: fluentd +data: + chart_name: fluentd + release: fluentd + namespace: osh-infra + wait: + timeout: 3600 + labels: + release_group: osh-infra-fluentd + resources: + - type: deployment + test: + timeout: 600 + install: + no_hooks: False + upgrade: + no_hooks: False + pre: + delete: + - type: job + labels: + release_group: osh-infra-fluentd + - type: pod + labels: + release_group: osh-infra-fluentd component: test values: release_uuid: ${RELEASE_UUID} @@ -384,9 +499,6 @@ data: fluentd: node_selector_key: openstack-control-plane node_selector_value: enabled - fluentbit: - node_selector_key: openstack-control-plane - node_selector_value: enabled prometheus_fluentd_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled @@ -396,7 +508,7 @@ data: source: type: local location: ${OSH_INFRA_PATH} - subpath: fluent-logging + subpath: fluentd reference: master dependencies: - helm-toolkit diff --git a/tools/deployment/common/fluentbit.sh b/tools/deployment/common/fluentbit.sh new file mode 100755 index 000000000..93c106878 --- /dev/null +++ b/tools/deployment/common/fluentbit.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make fluentbit + +helm upgrade --install fluentbit ./fluentbit \ + --namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status fluentbit diff --git a/tools/deployment/multinode/125-fluentbit.sh b/tools/deployment/multinode/125-fluentbit.sh new file mode 120000 index 000000000..0ed92806a --- /dev/null +++ b/tools/deployment/multinode/125-fluentbit.sh @@ -0,0 +1 @@ +../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/multinode/130-fluent-logging.sh b/tools/deployment/multinode/130-fluentd.sh similarity index 75% rename from tools/deployment/multinode/130-fluent-logging.sh rename to tools/deployment/multinode/130-fluentd.sh index 50bfe63e8..c1af05064 100755 --- a/tools/deployment/multinode/130-fluent-logging.sh +++ b/tools/deployment/multinode/130-fluentd.sh @@ -17,10 +17,10 @@ set -xe #NOTE: Lint and package chart -make fluent-logging +make fluentd if [ ! -d "/var/log/journal" ]; then -tee /tmp/fluent-logging.yaml << EOF +tee /tmp/fluentd.yaml << EOF monitoring: prometheus: enabled: true @@ -36,20 +36,16 @@ pod: - name: runlog mountPath: /run/log EOF -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ - --values=/tmp/fluent-logging.yaml + --values=/tmp/fluentd.yaml else -helm upgrade --install fluent-logging ./fluent-logging \ - --namespace=osh-infra \ - --set monitoring.prometheus.enabled=true +helm upgrade --install fluentd ./fluentd \ + --namespace=osh-infra fi #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh osh-infra #NOTE: Validate Deployment info -helm status fluent-logging - -#NOTE: Run helm tests -helm test fluent-logging +helm status fluentd diff --git a/tools/deployment/osh-infra-logging/060-fluentbit.sh b/tools/deployment/osh-infra-logging/060-fluentbit.sh new file mode 120000 index 000000000..0ed92806a --- /dev/null +++ b/tools/deployment/osh-infra-logging/060-fluentbit.sh @@ -0,0 +1 @@ +../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/060-fluent-logging.sh b/tools/deployment/osh-infra-logging/065-fluentd.sh similarity index 82% rename from tools/deployment/osh-infra-logging/060-fluent-logging.sh rename to tools/deployment/osh-infra-logging/065-fluentd.sh index ea95267df..e911d452a 100755 --- a/tools/deployment/osh-infra-logging/060-fluent-logging.sh +++ b/tools/deployment/osh-infra-logging/065-fluentd.sh @@ -17,10 +17,10 @@ set -xe #NOTE: Lint and package chart -make fluent-logging +make fluentd if [ ! -d "/var/log/journal" ]; then -tee /tmp/fluent-logging.yaml << EOF +tee /tmp/fluentd.yaml << EOF monitoring: prometheus: enabled: true @@ -38,11 +38,11 @@ pod: - name: runlog mountPath: /run/log EOF -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ - --values=/tmp/fluent-logging.yaml + --values=/tmp/fluentd.yaml else -helm upgrade --install fluent-logging ./fluent-logging \ +helm upgrade --install fluentd ./fluentd \ --namespace=osh-infra \ --set pod.replicas.fluentd=1 \ --set monitoring.prometheus.enabled=true @@ -52,6 +52,4 @@ fi ./tools/deployment/common/wait-for-pods.sh osh-infra #NOTE: Validate Deployment info -helm status fluent-logging - -helm test fluent-logging +helm status fluentd diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8d1cfe40e..e222c59ce 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -60,7 +60,8 @@ - ./tools/deployment/multinode/110-nagios.sh - ./tools/deployment/multinode/115-radosgw-osh-infra.sh - ./tools/deployment/multinode/120-elasticsearch.sh - - ./tools/deployment/multinode/130-fluent-logging.sh + - ./tools/deployment/multinode/125-fluentbit.sh + - ./tools/deployment/multinode/130-fluentd.sh - ./tools/deployment/multinode/140-kibana.sh - ./tools/deployment/multinode/600-grafana-selenium.sh - ./tools/deployment/multinode/610-nagios-selenium.sh @@ -127,7 +128,8 @@ - ./tools/deployment/osh-infra-logging/040-ldap.sh - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh - ./tools/deployment/osh-infra-logging/055-elasticsearch-ldap.sh - - ./tools/deployment/osh-infra-logging/060-fluent-logging.sh + - ./tools/deployment/osh-infra-logging/060-fluentbit.sh + - ./tools/deployment/osh-infra-logging/065-fluentd.sh - ./tools/deployment/osh-infra-logging/070-kibana.sh - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh