diff --git a/fluent-logging/templates/_helpers.tpl b/fluent-logging/templates/_helpers.tpl index 6cbf26ca9..c3dafbfd2 100644 --- a/fluent-logging/templates/_helpers.tpl +++ b/fluent-logging/templates/_helpers.tpl @@ -108,3 +108,34 @@ section): {{- end -}} {{- end -}} {{- end -}} + +# This function generates elasticsearch template files with entries in the +# fluent-logging values.yaml. It results in a configuration section with the +# following format (for as many key/value pairs defined in values for a section): +# { +# key: value +# key: { +# key: { ... } +# } +# } +# The configuration schema can be found here: +# https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html + +{{- define "fluent_logging.to_elasticsearch_template" -}} +{ +{{- include "fluent_logging.recursive_tuple" . | indent 2 }} +} +{{- end }} + +{{- define "fluent_logging.recursive_tuple" -}} +{{- range $key, $value := . -}} +, +{{- if or (kindIs "map" $value) }} +{{ $key | quote -}}:{ +{{- include "fluent_logging.recursive_tuple" $value | indent 2 }} +} +{{- else }} +{{ $key | quote -}}:{{ $value | quote }} +{{- end }} +{{- end }} +{{- end }} diff --git a/fluent-logging/templates/bin/_create_template.sh.tpl b/fluent-logging/templates/bin/_create_template.sh.tpl new file mode 100644 index 000000000..ec9cf348e --- /dev/null +++ b/fluent-logging/templates/bin/_create_template.sh.tpl @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +sed 's/ ,//' /tmp/template.xml.raw > /tmp/template.xml +result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ +-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/template_fluent_logging" \ +-H 'Content-Type: application/json' -d @/tmp/template.xml) +if [ "$result" == "True" ]; then + echo "template created!" +else + echo "template not created!" +fi diff --git a/fluent-logging/templates/bin/_helm-tests.sh.tpl b/fluent-logging/templates/bin/_helm-tests.sh.tpl index deb717b43..48ccb5419 100644 --- a/fluent-logging/templates/bin/_helm-tests.sh.tpl +++ b/fluent-logging/templates/bin/_helm-tests.sh.tpl @@ -46,7 +46,22 @@ function check_kubernetes_tag () { fi } +# Tests whether fluent-logging has successfully generate template_fluent_logging template +# defined by value.yaml +function check_template () { + total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/_template/template_fluent_logging" -H 'Content-Type: application/json' \ + | python -c "import sys, json; print len(json.load(sys.stdin))") + if [ "$total_hits" -gt 0 ]; then + echo "PASS: Successful hits on template, provided by fluent-logging!" + else + echo "FAIL: No hits on query for template_fluent_logging template! Exiting"; + exit 1; + fi +} + # Sleep for at least the buffer flush time to allow for indices to be populated sleep 30 +check_template check_logstash_index check_kubernetes_tag diff --git a/fluent-logging/templates/configmap-bin.yaml b/fluent-logging/templates/configmap-bin.yaml index 312f59af6..be40c2ccc 100644 --- a/fluent-logging/templates/configmap-bin.yaml +++ b/fluent-logging/templates/configmap-bin.yaml @@ -28,6 +28,8 @@ data: {{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_template.sh: | +{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: |+ {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/configmap-etc.yaml b/fluent-logging/templates/configmap-etc.yaml index 63d892909..029e1bd9e 100644 --- a/fluent-logging/templates/configmap-etc.yaml +++ b/fluent-logging/templates/configmap-etc.yaml @@ -28,4 +28,6 @@ data: {{ include "fluent_logging.to_fluentbit_conf" .Values.conf.parsers | indent 4 }} td-agent.conf: |+ {{ include "fluent_logging.to_fluentd_conf" .Values.conf.td_agent | indent 4 }} + template.xml.raw: |+ +{{ include "fluent_logging.to_elasticsearch_template" .Values.conf.template | indent 4 }} {{- end }} diff --git a/fluent-logging/templates/job-elasticsearch-template.yaml b/fluent-logging/templates/job-elasticsearch-template.yaml new file mode 100644 index 000000000..415f05d68 --- /dev/null +++ b/fluent-logging/templates/job-elasticsearch-template.yaml @@ -0,0 +1,74 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_elasticsearch_template }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.elasticsearch_template }} +{{- $mounts_elasticsearch_template := .Values.pod.mounts.elasticsearch_template.elasticsearch_template }} +{{- $mounts_elasticsearch_template_init := .Values.pod.mounts.elasticsearch_template.init_container }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: elasticsearch-template +spec: + template: + metadata: + labels: +{{ tuple $envAll "fluent" "elasticsearch-template" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.elasticsearch_template.node_selector_key }}: {{ .Values.labels.elasticsearch_template.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies $mounts_elasticsearch_template_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: elasticsearch-template + image: {{ .Values.images.tags.elasticsearch_template }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_template | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + value: {{ .Values.endpoints.elasticsearch.auth.admin.username }} + - name: ELASTICSEARCH_PASSWORD + value: {{ .Values.endpoints.elasticsearch.auth.admin.password }} + command: + - /tmp/create_template.sh + volumeMounts: + - name: fluent-logging-bin + mountPath: /tmp/create_template.sh + subPath: create_template.sh + readOnly: true + - name: fluent-logging-etc + mountPath: /tmp/template.xml.raw + subPath: template.xml.raw + readOnly: true +{{ if $mounts_elasticsearch_template.volumeMounts }}{{ toYaml $mounts_elasticsearch_template.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: fluent-logging-bin + configMap: + name: fluent-logging-bin + defaultMode: 0555 + - name: fluent-logging-etc + configMap: + name: fluent-logging-etc + defaultMode: 0666 +{{ if $mounts_elasticsearch_template.volumes }}{{ toYaml $mounts_elasticsearch_template.volumes | indent 6 }}{{ end }} +{{- end }} diff --git a/fluent-logging/values.yaml b/fluent-logging/values.yaml index 6a94f0a4b..f70593944 100644 --- a/fluent-logging/values.yaml +++ b/fluent-logging/values.yaml @@ -28,6 +28,9 @@ labels: prometheus_fluentd_exporter: node_selector_key: openstack-control-plane node_selector_value: enabled + elasticsearch_template: + node_selector_key: openstack-control-plane + node_selector_value: enabled images: tags: @@ -36,6 +39,7 @@ images: prometheus_fluentd_exporter: docker.io/srwilkers/fluentd_exporter:v0.1 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 + elasticsearch_template: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent local_registry: @@ -64,9 +68,15 @@ dependencies: - service: kafka endpoint: public fluentbit: + jobs: + - elasticsearch-template services: - service: fluentd endpoint: internal + elasticsearch-template: + services: + - service: elasticsearch + endpoint: internal tests: services: - service: elasticsearch @@ -156,6 +166,50 @@ conf: log: format: "logger:stdout?json=true" level: "info" + template: + template: "logstash-*" + settings: + number_of_shards: 5 + number_of_replicas: 1 + mappings: + flb_type: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: not_analyzed + docker_id: + type: keyword + index: not_analyzed + host: + type: keyword + index: not_analyzed + labels: + properties: + app: + type: keyword + index: not_analyzed + application: + type: keyword + index: not_analyzed + component: + type: keyword + index: not_analyzed + release_group: + type: keyword + index: not_analyzed + namespace_name: + type: keyword + index: not_analyzed + pod_id: + type: keyword + index: not_analyzed + pod_name: + type: keyword + index: not_analyzed + log: + type: text endpoints: cluster_domain_suffix: cluster.local @@ -319,6 +373,9 @@ pod: fluentbit: fluent_tests: fluent_tests: + elasticsearch_template: + init_container: + elasticsearch_template: manifests: configmap_bin: true @@ -334,3 +391,4 @@ manifests: service_exporter: true secret_elasticsearch: true service_fluentd: true + job_elasticsearch_template: true