From 2862f038e27a79baed04097ba4b112bd7c274a6e Mon Sep 17 00:00:00 2001 From: sungil Date: Tue, 14 Nov 2017 15:22:57 +0900 Subject: [PATCH] Fluent-logging helm chart This introduces an initial helm chart for fluent logging. It provides a functional fluent-bit and fluentd deployment to use in conjunction with elasticsearch and kibana to consume and aggregate logs from all resource types in a cluster. It can deliver logs to kafka for external tools to consume. This PS moves fluent-logging chart from osh-addons, osh to osh-infra repo. previous ps(addon): https://review.openstack.org/#/c/507023/ previous ps(osh): https://review.openstack.org/#/c/514622/ Specification: https://review.openstack.org/#/c/505491/ Partially implements: blueprint osh-logging-framework Change-Id: I72e580aa3a197550060fc07af8396a7c8368d40b --- fluent-logging/Chart.yaml | 25 ++ fluent-logging/README.rst | 30 +++ fluent-logging/requirements.yaml | 19 ++ fluent-logging/templates/bin/_fluentd.sh.tpl | 30 +++ .../templates/bin/_helm-tests.sh.tpl | 50 ++++ fluent-logging/templates/clusterrole.yaml | 54 ++++ .../templates/clusterrolebinding.yaml | 30 +++ fluent-logging/templates/configmap-bin.yaml | 31 +++ fluent-logging/templates/configmap-etc.yaml | 31 +++ .../templates/daemonset-fluent-bit.yaml | 82 +++++++ .../templates/deployment-fluentd.yaml | 84 +++++++ .../templates/etc/_fluent-bit.conf.tpl | 19 ++ .../templates/etc/_parsers.conf.tpl | 6 + .../templates/etc/_td-agent.conf.tpl | 83 +++++++ .../templates/job-image-repo-sync.yaml | 65 +++++ fluent-logging/templates/pod-helm-tests.yaml | 46 ++++ fluent-logging/templates/rbac-entrypoint.yaml | 19 ++ fluent-logging/templates/service-fluentd.yaml | 37 +++ fluent-logging/templates/serviceaccount.yaml | 22 ++ fluent-logging/values.yaml | 230 ++++++++++++++++++ tools/gate/chart-deploys/default.yaml | 11 + 21 files changed, 1004 insertions(+) create mode 100644 fluent-logging/Chart.yaml create mode 100644 fluent-logging/README.rst create mode 100644 fluent-logging/requirements.yaml create mode 100644 fluent-logging/templates/bin/_fluentd.sh.tpl create mode 100644 fluent-logging/templates/bin/_helm-tests.sh.tpl create mode 100644 fluent-logging/templates/clusterrole.yaml create mode 100644 fluent-logging/templates/clusterrolebinding.yaml create mode 100644 fluent-logging/templates/configmap-bin.yaml create mode 100644 fluent-logging/templates/configmap-etc.yaml create mode 100644 fluent-logging/templates/daemonset-fluent-bit.yaml create mode 100644 fluent-logging/templates/deployment-fluentd.yaml create mode 100644 fluent-logging/templates/etc/_fluent-bit.conf.tpl create mode 100644 fluent-logging/templates/etc/_parsers.conf.tpl create mode 100644 fluent-logging/templates/etc/_td-agent.conf.tpl create mode 100644 fluent-logging/templates/job-image-repo-sync.yaml create mode 100644 fluent-logging/templates/pod-helm-tests.yaml create mode 100644 fluent-logging/templates/rbac-entrypoint.yaml create mode 100644 fluent-logging/templates/service-fluentd.yaml create mode 100644 fluent-logging/templates/serviceaccount.yaml create mode 100644 fluent-logging/values.yaml diff --git a/fluent-logging/Chart.yaml b/fluent-logging/Chart.yaml new file mode 100644 index 000000000..e87238067 --- /dev/null +++ b/fluent-logging/Chart.yaml @@ -0,0 +1,25 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Fluentd +name: fluent-logging +version: 0.1.0 +home: http://www.fluentbit.io/ +sources: + - https://github.com/fluent/fluentbit + - https://github.com/fluent/fluentd + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/fluent-logging/README.rst b/fluent-logging/README.rst new file mode 100644 index 000000000..375a70414 --- /dev/null +++ b/fluent-logging/README.rst @@ -0,0 +1,30 @@ +Fluentd-logging +=============== + +OpenStack-Helm defines a centralized logging mechanism to provide insight into +the the state of the OpenStack services and infrastructure components as +well as underlying kubernetes platform. Among the requirements for a logging +platform, where log data can come from and where log data need to be delivered +are very variable. To support various logging scenarios, OpenStack-Helm should +provide a flexible mechanism to meet with certain operation needs. This chart +proposes fast and lightweight log forwarder and full featured log aggregator +complementing each other providing a flexible and reliable solution. Especially, +Fluent-bit is proposed as a log forwarder and Fluentd is proposed as a main log +aggregator and processor. + + +Mechanism +--------- + +Fluent-bit, Fluentd meet OpenStack-Helm's logging requirements for gathering, +aggregating, and delivering of logged events. Flunt-bit runs as a daemonset on +each node and mounts the /var/lib/docker/containers directory. The Docker +container runtime engine directs events posted to stdout and stderr to this +directory on the host. Fluent-bit then forward the contents of that directory to +Fluentd. Fluentd runs as deployment at the designated nodes and expose service +for Fluent-bit to foward logs. Fluentd should then apply the Logstash format to +the logs. Fluentd can also write kubernetes and OpenStack metadata to the logs. +Fluentd will then forward the results to Elasticsearch and to optionally kafka. +Elasticsearch indexes the logs in a logstash-* index by default. kafka stores +the logs in a 'logs' topic by default. Any external tool can then consume the +'logs' topic. diff --git a/fluent-logging/requirements.yaml b/fluent-logging/requirements.yaml new file mode 100644 index 000000000..00b2a9554 --- /dev/null +++ b/fluent-logging/requirements.yaml @@ -0,0 +1,19 @@ + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/fluent-logging/templates/bin/_fluentd.sh.tpl b/fluent-logging/templates/bin/_fluentd.sh.tpl new file mode 100644 index 000000000..0450572c1 --- /dev/null +++ b/fluent-logging/templates/bin/_fluentd.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec /usr/sbin/td-agent +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 000000000..304dee0de --- /dev/null +++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,50 @@ +#!/bin/bash + +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +# Tests whether fluentd has successfully indexed data into Elasticsearch under +# the logstash-* index via the fluent-elasticsearch plugin +function check_logstash_index () { + total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \ + | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") + if [ "$total_hits" -gt 0 ]; then + echo "PASS: Successful hits on logstash-* index, provided by fluentd!" + else + echo "FAIL: No hits on query for logstash-* index! Exiting"; + exit 1; + fi +} + +# Tests whether fluentd has successfully tagged data with the kube.* +# prefix via the fluent-kubernetes plugin +function check_kubernetes_tag () { + total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}/logstash-*/fluentd/_search?q=tag:kube.*" -H 'Content-Type: application/json' \ + | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") + if [ "$total_hits" -gt 0 ]; then + echo "PASS: Successful hits on logstash-* index, provided by fluentd!" + else + echo "FAIL: No hits on query for logstash-* index! Exiting"; + exit 1; + fi +} + +# Sleep for at least the buffer flush time to allow for indices to be populated +sleep 30 +check_logstash_index +check_kubernetes_tag diff --git a/fluent-logging/templates/clusterrole.yaml b/fluent-logging/templates/clusterrole.yaml new file mode 100644 index 000000000..7fe755db9 --- /dev/null +++ b/fluent-logging/templates/clusterrole.yaml @@ -0,0 +1,54 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.clusterrole }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: fluent-logging-runner +rules: +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/fluent-logging/templates/clusterrolebinding.yaml b/fluent-logging/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..4d8f32005 --- /dev/null +++ b/fluent-logging/templates/clusterrolebinding.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.clusterrolebinding }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: run-fluent-logging +subjects: + - kind: ServiceAccount + name: fluent-logging + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: fluent-logging-runner + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/fluent-logging/templates/configmap-bin.yaml b/fluent-logging/templates/configmap-bin.yaml new file mode 100644 index 000000000..d95622d77 --- /dev/null +++ b/fluent-logging/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-logging-bin +data: + fluentd.sh: | +{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: |+ +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml new file mode 100644 index 000000000..75f46b8a6 --- /dev/null +++ b/fluent-logging/templates/configmap-etc.yaml @@ -0,0 +1,31 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluent-logging-etc +data: + fluent-bit.conf: |+ +{{- tuple .Values.conf.fluentbit "etc/_fluent-bit.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} + parsers.conf: |+ +{{- tuple .Values.conf.parsers "etc/_parsers.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} + td-agent.conf: |+ +{{- tuple .Values.conf.td_agent "etc/_td-agent.conf.tpl" . | include "helm-toolkit.utils.configmap_templater" }} +{{- end }} diff --git a/fluent-logging/templates/daemonset-fluent-bit.yaml b/fluent-logging/templates/daemonset-fluent-bit.yaml new file mode 100644 index 000000000..640b58f46 --- /dev/null +++ b/fluent-logging/templates/daemonset-fluent-bit.yaml @@ -0,0 +1,82 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_fluentbit }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.fluentbit }} +{{- if .Values.images.local_registry.active -}} +{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentbit .Values.conditional_dependencies.local_image_registry) -}} +{{- else -}} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentbit -}} +{{- end -}} +{{- $mounts_fluentbit := .Values.pod.mounts.fluentbit.fluentbit }} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: fluentbit +spec: +{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "fluentbit" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccount: fluent-logging + nodeSelector: + {{ .Values.labels.fluentbit.node_selector_key }}: {{ .Values.labels.fluentbit.node_selector_value }} + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: fluentbit + env: + image: {{ .Values.images.tags.fluentbit }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: fluent-logging-etc + mountPath: /fluent-bit/etc/fluent-bit.conf + subPath: fluent-bit.conf + readOnly: true + - name: fluent-logging-etc + mountPath: /fluent-bit/etc/parsers.conf + subPath: parsers.conf + readOnly: true +{{ if $mounts_fluentbit.volumeMounts }}{{ toYaml $mounts_fluentbit.volumeMounts | indent 8 }}{{ end }} + volumes: +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: fluent-logging-etc + configMap: + name: fluent-logging-etc + defaultMode: 0444 +{{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/fluent-logging/templates/deployment-fluentd.yaml b/fluent-logging/templates/deployment-fluentd.yaml new file mode 100644 index 000000000..4bc84ac8a --- /dev/null +++ b/fluent-logging/templates/deployment-fluentd.yaml @@ -0,0 +1,84 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment_fluentd }} +{{- $envAll := . }} +{{- if .Values.images.local_registry.active -}} +{{- $_ := set .Values "pod_dependency" (merge .Values.dependencies.fluentd .Values.conditional_dependencies.local_image_registry) -}} +{{- else -}} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.fluentd -}} +{{- end -}} +{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: fluentd +spec: + replicas: {{ .Values.pod.replicas.fluentd }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccount: fluent-logging + affinity: +{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd_aggregator.timeout | default "30" }} + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: fluentd + image: {{ .Values.images.tags.fluentd }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/fluentd.sh + - start + ports: + - containerPort: {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + volumeMounts: + - name: pod-etc-fluentd + mountPath: /etc/td-agent + - name: fluent-logging-etc + mountPath: /etc/td-agent/td-agent.conf + subPath: td-agent.conf + readOnly: true + - name: fluent-logging-bin + mountPath: /tmp/fluentd.sh + subPath: fluentd.sh + readOnly: true +{{- if $mounts_fluentd.volumeMounts }}{{ toYaml $mounts_fluentd.volumeMounts | indent 12 }}{{- end }} + volumes: +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} + - name: pod-etc-fluentd + emptyDir: {} + - name: fluent-logging-etc + configMap: + name: fluent-logging-etc + defaultMode: 0444 + - name: fluent-logging-bin + configMap: + name: fluent-logging-bin + defaultMode: 0555 +{{- if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }} +{{- end }} diff --git a/fluent-logging/templates/etc/_fluent-bit.conf.tpl b/fluent-logging/templates/etc/_fluent-bit.conf.tpl new file mode 100644 index 000000000..7b09615d9 --- /dev/null +++ b/fluent-logging/templates/etc/_fluent-bit.conf.tpl @@ -0,0 +1,19 @@ +[SERVICE] + Flush 1 + Daemon Off + Log_Level {{ .Values.conf.fluentbit.service.log_level }} + Parsers_File parsers.conf + +[INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser docker + DB /var/log/flb_kube.db + Mem_Buf_Limit {{ .Values.conf.fluentbit.input.mem_buf_limit }} + +[OUTPUT] + Name forward + Match * + Host {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + Port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} diff --git a/fluent-logging/templates/etc/_parsers.conf.tpl b/fluent-logging/templates/etc/_parsers.conf.tpl new file mode 100644 index 000000000..9f3b6b331 --- /dev/null +++ b/fluent-logging/templates/etc/_parsers.conf.tpl @@ -0,0 +1,6 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On diff --git a/fluent-logging/templates/etc/_td-agent.conf.tpl b/fluent-logging/templates/etc/_td-agent.conf.tpl new file mode 100644 index 000000000..b9d78bbb2 --- /dev/null +++ b/fluent-logging/templates/etc/_td-agent.conf.tpl @@ -0,0 +1,83 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + + @type forward + port {{ tuple "aggregator" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + bind 0.0.0.0 + + + + type kubernetes_metadata + + + +{{ if .Values.conf.fluentd.kafka.enabled }} + @type copy + + + @type kafka_buffered + + # list of seed brokers + brokers {{ tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "kafka" "public" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + # buffer settings + buffer_type file + buffer_path /var/log/td-agent/buffer/td + flush_interval {{ .Values.conf.fluentd.kafka.flush_interval }} + + # topic settings + default_topic {{ .Values.conf.fluentd.kafka.topic_name }} + + # data type settings + output_data_type {{ .Values.conf.fluentd.kafka.output_data_type }} + compression_codec gzip + + # producer settings + max_send_retries 1 + required_acks -1 + + + +{{- end }} + @type elasticsearch + include_tag_key true + host {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + port {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + logstash_format {{ .Values.conf.fluentd.elasticsearch.logstash }} + + # Set the chunk limit the same as for fluentd-gcp. + buffer_chunk_limit {{ .Values.conf.fluentd.elasticsearch.buffer_chunk_limit }} + + # Cap buffer memory usage to 2MiB/chunk * 32 chunks = 64 MiB + buffer_queue_limit {{ .Values.conf.fluentd.elasticsearch.buffer_queue_limit }} + + # Flush buffer every 30s to write to Elasticsearch + flush_interval {{ .Values.conf.fluentd.elasticsearch.flush_interval }} + + # Never wait longer than 5 minutes between retries. + max_retry_wait {{ .Values.conf.fluentd.elasticsearch.max_retry_wait }} + +{{- if .Values.conf.fluentd.elasticsearch.disable_retry_limit }} + + # Disable the limit on the number of retries (retry forever). + disable_retry_limit +{{- end }} + + # Use multiple threads for processing. + num_threads {{ .Values.conf.fluentd.elasticsearch.num_threads }} +{{ if .Values.conf.fluentd.kafka.enabled }} + +{{- end }} + + diff --git a/fluent-logging/templates/job-image-repo-sync.yaml b/fluent-logging/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..741d93616 --- /dev/null +++ b/fluent-logging/templates/job-image-repo-sync.yaml @@ -0,0 +1,65 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_image_repo_sync }} +{{- $envAll := . }} +{{- if .Values.images.local_registry.active -}} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.image_repo_sync -}} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: fluent-logging-image-repo-sync +spec: + template: + metadata: + labels: +{{ tuple $envAll "fluent-logging-exporter" "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: image-repo-sync +{{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.image_repo_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: LOCAL_REPO + value: "{{ tuple "local_image_registry" "node" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "local_image_registry" "node" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: IMAGE_SYNC_LIST + value: "{{ include "helm-toolkit.utils.image_sync_list" . }}" + command: + - /tmp/image-repo-sync.sh + volumeMounts: + - name: fluent-logging-exporter-bin + mountPath: /tmp/image-repo-sync.sh + subPath: image-repo-sync.sh + readOnly: true + - name: docker-socket + mountPath: /var/run/docker.sock + volumes: +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_secret_mount" | indent 8 }} + - name: fluent-logging-bin + configMap: + name: fluent-logging-bin + defaultMode: 0555 + - name: docker-socket + hostPath: + path: /var/run/docker.sock +{{- end }} +{{- end }} diff --git a/fluent-logging/templates/pod-helm-tests.yaml b/fluent-logging/templates/pod-helm-tests.yaml new file mode 100644 index 000000000..98349f052 --- /dev/null +++ b/fluent-logging/templates/pod-helm-tests.yaml @@ -0,0 +1,46 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + annotations: + "helm.sh/hook": test-success +spec: + restartPolicy: Never + containers: + - name: {{.Release.Name}}-helm-tests +{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} + command: + - /tmp/helm-tests.sh + env: + - name: ELASTICSEARCH_ENDPOINT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + volumeMounts: + - name: fluent-logging-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true + volumes: + - name: fluent-logging-bin + configMap: + name: fluent-logging-bin + defaultMode: 0555 +{{- end }} diff --git a/fluent-logging/templates/rbac-entrypoint.yaml b/fluent-logging/templates/rbac-entrypoint.yaml new file mode 100644 index 000000000..311712ea9 --- /dev/null +++ b/fluent-logging/templates/rbac-entrypoint.yaml @@ -0,0 +1,19 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.rbac_entrypoint }} +{{ tuple . | include "helm-toolkit.snippets.kubernetes_entrypoint_rbac"}} +{{- end }} diff --git a/fluent-logging/templates/service-fluentd.yaml b/fluent-logging/templates/service-fluentd.yaml new file mode 100644 index 000000000..4a3aa63bb --- /dev/null +++ b/fluent-logging/templates/service-fluentd.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_fluentd }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "aggregator" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: aggregator + port: {{ .Values.network.fluentd.port }} + {{ if .Values.network.fluentd.node_port.enabled }} + nodePort: {{ .Values.network.fluentd.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "aggregator" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.fluentd.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} + diff --git a/fluent-logging/templates/serviceaccount.yaml b/fluent-logging/templates/serviceaccount.yaml new file mode 100644 index 000000000..8d09a19c1 --- /dev/null +++ b/fluent-logging/templates/serviceaccount.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.serviceaccount }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: fluent-logging +{{- end }} diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml new file mode 100644 index 000000000..995c011fd --- /dev/null +++ b/fluent-logging/values.yaml @@ -0,0 +1,230 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for fluentbit. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + fluentd: + node_selector_key: openstack-control-plane + node_selector_value: enabled + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + fluentbit: docker.io/fluent/fluent-bit:0.12.9 + fluentd: docker.io/kolla/ubuntu-source-fluentd:ocata + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + image_repo_sync: + services: + - service: local_image_registry + endpoint: internal + fluentd: + services: + - service: elasticsearch + endpoint: internal + fluentd_with_kafka: + services: + - service: elasticsearch + endpoint: internal + - service: kafka + endpoint: public + fluentbit: + services: + - service: aggregator + endpoint: internal + tests: + services: + - service: elasticsearch + endpoint: internal + - service: aggregator + endpoint: internal + +conditional_dependencies: + local_image_registry: + jobs: + - fluent-logging-image-repo-sync + services: + - service: local_image_registry + endpoint: node + fluentd: + services: + - service: kafka + endpoint: public + + +conf: + fluentbit: + service: + log_level: info + input: + mem_buf_limit: 5MB + override: + fluentd: + kafka: + enabled: false + topic_name: logs + flush_interval: 3s + output_data_type: json + elasticsearch: + logstash: true + buffer_chunk_limit: 10M + buffer_queue_limit: 32 + flush_interval: 15s + max_retry_wait: 300 + disable_retry_limit: true + num_threads: 8 + override: + +endpoints: + cluster_domain_suffix: cluster.local + elasticsearch: + namespace: null + name: elasticsearch + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + client: + default: 9200 + discovery: + default: 9300 + kafka: + namespace: null + name: kafka + hosts: + default: kafka-logging + public: kafka + scheme: + default: http + public: http + port: + service: + default: 9092 + aggregator: + namespace: null + name: fluentd + hosts: + default: fluentd-logging + internal: fluentd-logging + scheme: + default: http + port: + service: + default: 24224 + internal: 24224 + host_fqdn_override: + default: null + +network: + fluentd: + node_port: + enabled: false + port: 32329 + port: 24224 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + fluentbit: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + fluentd_aggregator: + timeout: 30 + resources: + fluentbit: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + fluentd: + enabled: false + limits: + memory: '1024Mi' + cpu: '2000m' + requests: + memory: '128Mi' + cpu: '500m' + jobs: + tests: + limits: + memory: '1024Mi' + cpu: '2000m' + requests: + memory: '128Mi' + cpu: '100m' + replicas: + fluentd: 3 + mounts: + fluentd: + fluentd: + fluentbit: + fluentbit: + fluent_tests: + fluent_tests: + +manifests: + service_fluentd: true + deployment_fluentd: true + daemonset_fluentbit: true + job_image_repo_sync: true + helm_tests: true + configmap_bin: true + configmap_etc: true + clusterrole: true + clusterrolebinding: true + rbac_entrypoint: true + serviceaccount: true diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml index 3fb79c097..c356ae310 100644 --- a/tools/gate/chart-deploys/default.yaml +++ b/tools/gate/chart-deploys/default.yaml @@ -32,6 +32,7 @@ chart_groups: timeout: 600 charts: - openstack_elasticsearch + - fluent_logging charts: docker_registry_nfs_provisioner: @@ -140,3 +141,13 @@ charts: values: storage: enabled: false + + fluent_logging: + chart_name: fluent-logging + release: fluent-logging + namespace: openstack + timeout: 300 + test: + enabled: true + timeout: 300 + output: false