From 4c18a421ee1dfe934542563323d79f24fd4d8d02 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 6 Jul 2018 10:33:10 -0500 Subject: [PATCH] Add charts for Elastic Beats This adds basic charts for Elastic metricbeat, filebeat, packetbeat, and elastic APM server. This also adds an experimental job for deploying the elastic beats along with Elasticsearch and Kibana Change-Id: Idcdc1bfa75bcdcaa68801dbb8999f0853652af0f --- elastic-apm-server/Chart.yaml | 24 ++ elastic-apm-server/requirements.yaml | 18 ++ .../templates/configmap-bin.yaml | 27 ++ .../templates/configmap-etc.yaml | 27 ++ elastic-apm-server/templates/deployment.yaml | 123 +++++++++ .../templates/job-image-repo-sync.yaml | 20 ++ .../templates/secret-elasticsearch-creds.yaml | 29 +++ elastic-apm-server/templates/service.yaml | 34 +++ elastic-apm-server/values.yaml | 166 ++++++++++++ elastic-filebeat/Chart.yaml | 24 ++ elastic-filebeat/requirements.yaml | 18 ++ elastic-filebeat/templates/configmap-bin.yaml | 27 ++ elastic-filebeat/templates/configmap-etc.yaml | 27 ++ elastic-filebeat/templates/daemonset.yaml | 150 +++++++++++ .../templates/job-image-repo-sync.yaml | 20 ++ .../templates/secret-elasticsearch-creds.yaml | 29 +++ elastic-filebeat/values.yaml | 216 ++++++++++++++++ elastic-metricbeat/Chart.yaml | 24 ++ elastic-metricbeat/requirements.yaml | 18 ++ .../templates/configmap-etc.yaml | 37 +++ .../templates/daemonset-node-metrics.yaml | 165 ++++++++++++ .../templates/deployment-modules.yaml | 106 ++++++++ .../templates/job-image-repo-sync.yaml | 20 ++ .../templates/secret-elasticsearch-creds.yaml | 29 +++ elastic-metricbeat/values.yaml | 243 ++++++++++++++++++ elastic-packetbeat/Chart.yaml | 24 ++ elastic-packetbeat/requirements.yaml | 18 ++ .../templates/configmap-etc.yaml | 27 ++ elastic-packetbeat/templates/daemonset.yaml | 144 +++++++++++ .../templates/job-image-repo-sync.yaml | 20 ++ .../templates/secret-elasticsearch-creds.yaml | 29 +++ elastic-packetbeat/values.yaml | 182 +++++++++++++ .../010-deploy-docker-registry.sh | 1 + tools/deployment/elastic-beats/020-ingress.sh | 1 + tools/deployment/elastic-beats/030-ceph.sh | 1 + .../elastic-beats/035-ceph-ns-activate.sh | 1 + tools/deployment/elastic-beats/040-ldap.sh | 1 + .../elastic-beats/050-elasticsearch.sh | 40 +++ .../elastic-beats/060-elastic-metricbeat.sh | 32 +++ .../elastic-beats/070-kube-state-metrics.sh | 1 + .../elastic-beats/080-elastic-filebeat.sh | 31 +++ .../elastic-beats/090-elastic-packetbeat.sh | 32 +++ .../elastic-beats/100-elastic-apm-server.sh | 31 +++ tools/deployment/elastic-beats/110-kibana.sh | 1 + zuul.d/jobs.yaml | 27 ++ zuul.d/project.yaml | 1 + 46 files changed, 2266 insertions(+) create mode 100644 elastic-apm-server/Chart.yaml create mode 100644 elastic-apm-server/requirements.yaml create mode 100644 elastic-apm-server/templates/configmap-bin.yaml create mode 100644 elastic-apm-server/templates/configmap-etc.yaml create mode 100644 elastic-apm-server/templates/deployment.yaml create mode 100644 elastic-apm-server/templates/job-image-repo-sync.yaml create mode 100644 elastic-apm-server/templates/secret-elasticsearch-creds.yaml create mode 100644 elastic-apm-server/templates/service.yaml create mode 100644 elastic-apm-server/values.yaml create mode 100644 elastic-filebeat/Chart.yaml create mode 100644 elastic-filebeat/requirements.yaml create mode 100644 elastic-filebeat/templates/configmap-bin.yaml create mode 100644 elastic-filebeat/templates/configmap-etc.yaml create mode 100644 elastic-filebeat/templates/daemonset.yaml create mode 100644 elastic-filebeat/templates/job-image-repo-sync.yaml create mode 100644 elastic-filebeat/templates/secret-elasticsearch-creds.yaml create mode 100644 elastic-filebeat/values.yaml create mode 100644 elastic-metricbeat/Chart.yaml create mode 100644 elastic-metricbeat/requirements.yaml create mode 100644 elastic-metricbeat/templates/configmap-etc.yaml create mode 100644 elastic-metricbeat/templates/daemonset-node-metrics.yaml create mode 100644 elastic-metricbeat/templates/deployment-modules.yaml create mode 100644 elastic-metricbeat/templates/job-image-repo-sync.yaml create mode 100644 elastic-metricbeat/templates/secret-elasticsearch-creds.yaml create mode 100644 elastic-metricbeat/values.yaml create mode 100644 elastic-packetbeat/Chart.yaml create mode 100644 elastic-packetbeat/requirements.yaml create mode 100644 elastic-packetbeat/templates/configmap-etc.yaml create mode 100644 elastic-packetbeat/templates/daemonset.yaml create mode 100644 elastic-packetbeat/templates/job-image-repo-sync.yaml create mode 100644 elastic-packetbeat/templates/secret-elasticsearch-creds.yaml create mode 100644 elastic-packetbeat/values.yaml create mode 120000 tools/deployment/elastic-beats/010-deploy-docker-registry.sh create mode 120000 tools/deployment/elastic-beats/020-ingress.sh create mode 120000 tools/deployment/elastic-beats/030-ceph.sh create mode 120000 tools/deployment/elastic-beats/035-ceph-ns-activate.sh create mode 120000 tools/deployment/elastic-beats/040-ldap.sh create mode 100755 tools/deployment/elastic-beats/050-elasticsearch.sh create mode 100755 tools/deployment/elastic-beats/060-elastic-metricbeat.sh create mode 120000 tools/deployment/elastic-beats/070-kube-state-metrics.sh create mode 100755 tools/deployment/elastic-beats/080-elastic-filebeat.sh create mode 100755 tools/deployment/elastic-beats/090-elastic-packetbeat.sh create mode 100755 tools/deployment/elastic-beats/100-elastic-apm-server.sh create mode 120000 tools/deployment/elastic-beats/110-kibana.sh diff --git a/elastic-apm-server/Chart.yaml b/elastic-apm-server/Chart.yaml new file mode 100644 index 000000000..6eac8ade2 --- /dev/null +++ b/elastic-apm-server/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic APM Server +name: elastic-apm-server +version: 0.1.0 +home: https://www.elastic.co/guide/en/apm/get-started/current/index.html +sources: + - https://github.com/elastic/apm-server + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-apm-server/requirements.yaml b/elastic-apm-server/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-apm-server/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-apm-server/templates/configmap-bin.yaml b/elastic-apm-server/templates/configmap-bin.yaml new file mode 100644 index 000000000..d26f95872 --- /dev/null +++ b/elastic-apm-server/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/configmap-etc.yaml b/elastic-apm-server/templates/configmap-etc.yaml new file mode 100644 index 000000000..6d584d02c --- /dev/null +++ b/elastic-apm-server/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-etc +data: + apm-server.yml: | +{{ toYaml .Values.conf.apm_server | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/deployment.yaml b/elastic-apm-server/templates/deployment.yaml new file mode 100644 index 000000000..19ff4fffc --- /dev/null +++ b/elastic-apm-server/templates/deployment.yaml @@ -0,0 +1,123 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_elastic_apm_server := .Values.pod.mounts.elastic_apm_server.elastic_apm_server }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elastic-apm-server" }} +{{ tuple $envAll "elastic-apm-server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +- apiGroups: [""] + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: elastic-apm-server +spec: +{{ tuple $envAll "elastic-apm-server" | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + replicas: {{ .Values.pod.replicas.elastic_apm_server }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.elastic_apm_server.node_selector_key }}: {{ .Values.labels.elastic_apm_server.node_selector_value }} + initContainers: +{{ tuple $envAll "elastic_apm_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: elastic-apm-server + image: {{ .Values.images.tags.elastic_apm_server }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.elastic_apm_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/usr/share/apm-server/apm-server.yml" + - "-e" + ports: + - name: server + containerPort: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: APM_SERVER_HOST + value: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: APM_SERVER_PORT + value: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: elastic-apm-server-etc + mountPath: /usr/share/apm-server/apm-server.yml + readOnly: true + subPath: apm-server.yml + - name: data + mountPath: /usr/share/apm-server/data +{{ if $mounts_elastic_apm_server.volumeMounts }}{{ toYaml $mounts_elastic_apm_server.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: elastic-apm-server-etc + configMap: + name: elastic-apm-server-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/elastic-apm-server + type: DirectoryOrCreate +{{ if $mounts_elastic_apm_server.volumes }}{{ toYaml $mounts_elastic_apm_server.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-apm-server/templates/job-image-repo-sync.yaml b/elastic-apm-server/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..cb701d04c --- /dev/null +++ b/elastic-apm-server/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-apm-server/templates/secret-elasticsearch-creds.yaml b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-apm-server/templates/service.yaml b/elastic-apm-server/templates/service.yaml new file mode 100644 index 000000000..d4aed6554 --- /dev/null +++ b/elastic-apm-server/templates/service.yaml @@ -0,0 +1,34 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: server + port: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + nodePort: {{ .Values.network.elastic_apm_server.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + type: NodePort + {{ end }} diff --git a/elastic-apm-server/values.yaml b/elastic-apm-server/values.yaml new file mode 100644 index 000000000..bc1f94b05 --- /dev/null +++ b/elastic-apm-server/values.yaml @@ -0,0 +1,166 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elastic-apm-server +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + elastic_apm_server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + elastic_apm_server: docker.elastic.co/apm/apm-server:6.2.3 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: elastic-apm-server-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - elastic-apm-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + elastic_apm_server: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + apm_server: + setup: + dashboards: + enabled: true + host: ['${APM_SERVER_HOST}:${APM_SERVER_PORT}'] + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + elastic_apm_server: + namespace: null + name: apm-server + hosts: + default: apm-server + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + server: + default: 8200 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + elastic_apm_server: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + elastic_apm_server: 1 + resources: + elastic_apm_server: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + elastic_apm_server: + elastic_apm_server: + +network: + elastic_apm_server: + node_port: + enabled: false + port: 30200 + +manifests: + configmap_bin: true + configmap_etc: true + deployment: true + service: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-filebeat/Chart.yaml b/elastic-filebeat/Chart.yaml new file mode 100644 index 000000000..3543e920c --- /dev/null +++ b/elastic-filebeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Filebeat +name: elastic-filebeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/filebeat +sources: + - https://github.com/elastic/beats/tree/master/filebeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-filebeat/requirements.yaml b/elastic-filebeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-filebeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-filebeat/templates/configmap-bin.yaml b/elastic-filebeat/templates/configmap-bin.yaml new file mode 100644 index 000000000..149f7fff4 --- /dev/null +++ b/elastic-filebeat/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..2e2fc1232 --- /dev/null +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-etc +data: + filebeat.yml: | +{{ toYaml .Values.conf | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml new file mode 100644 index 000000000..655307e94 --- /dev/null +++ b/elastic-filebeat/templates/daemonset.yaml @@ -0,0 +1,150 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_filebeat := .Values.pod.mounts.filebeat.filebeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "filebeat" }} +{{ tuple $envAll "filebeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat +spec: +{{ tuple $envAll "filebeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "filebeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.filebeat.node_selector_key }}: {{ .Values.labels.filebeat.node_selector_value }} + initContainers: +{{ tuple $envAll "filebeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: filebeat + image: {{ .Values.images.tags.filebeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.filebeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-e" + ports: + - name: filebeat + containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: data + mountPath: /usr/share/filebeat/data + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: filebeat-etc + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true + subPath: filebeat.yml +{{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: filebeat-etc + configMap: + name: filebeat-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/filebeat + type: DirectoryOrCreate +{{ if $mounts_filebeat.volumes }}{{ toYaml $mounts_filebeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-filebeat/templates/job-image-repo-sync.yaml b/elastic-filebeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..cb701d04c --- /dev/null +++ b/elastic-filebeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-filebeat/templates/secret-elasticsearch-creds.yaml b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml new file mode 100644 index 000000000..00cb6b406 --- /dev/null +++ b/elastic-filebeat/values.yaml @@ -0,0 +1,216 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for filebeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + filebeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + filebeat: docker.elastic.co/beats/filebeat:6.2.3 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: filebeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - filebeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + filebeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + setup: + dashboards: + enabled: true + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + prospectors: + path: ${path.config}/prospectors.d/*.yml + reload: + enabled: false + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: false + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + prospectors: + - type: docker + containers.ids: + - "*" + multiline: + pattern: '^Traceback' + match: after + negate: true + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + filebeat: + namespace: null + name: filebeat + hosts: + default: filebeat + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 5066 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + filebeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + filebeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + filebeat: + filebeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-metricbeat/Chart.yaml b/elastic-metricbeat/Chart.yaml new file mode 100644 index 000000000..0f4d40c94 --- /dev/null +++ b/elastic-metricbeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Metricbeat +name: elastic-metricbeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/metricbeat +sources: + - https://github.com/elastic/beats/tree/master/metricbeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-metricbeat/requirements.yaml b/elastic-metricbeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-metricbeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-metricbeat/templates/configmap-etc.yaml b/elastic-metricbeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..bada69fcc --- /dev/null +++ b/elastic-metricbeat/templates/configmap-etc.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-etc +data: + metricbeat.yml: | +{{ toYaml .Values.conf.metricbeat | indent 4 }} + rabbitmq.yml: | +{{ toYaml .Values.conf.modules.rabbitmq | indent 4 }} + mysql.yml: | +{{ toYaml .Values.conf.modules.mysql | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} + daemonset_kubernetes.yml: | +{{ toYaml .Values.conf.modules.daemonset_kubernetes | indent 4 }} + deployment_kubernetes.yml: | +{{ toYaml .Values.conf.modules.deployment_kubernetes | indent 4 }} +{{- end }} diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml new file mode 100644 index 000000000..64c6e091b --- /dev/null +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -0,0 +1,165 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_metricbeat := .Values.pod.mounts.metricbeat.metricbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat-node-modules +spec: +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + privileged: true + runAsUser: 0 + image: {{ .Values.images.tags.metricbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/etc/metricbeat.yml" + - "-e" + - "-system.hostfs=/hostfs" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: metricbeat-etc + mountPath: /etc/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/system.yml + subPath: system.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: daemonset_kubernetes.yml + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true +{{ if $mounts_metricbeat.volumeMounts }}{{ toYaml $mounts_metricbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: metricbeat-etc + configMap: + defaultMode: 0444 + name: metricbeat-etc + - name: data + emptyDir: {} +{{ if $mounts_metricbeat.volumes }}{{ toYaml $mounts_metricbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml new file mode 100644 index 000000000..e4162019b --- /dev/null +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -0,0 +1,106 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "metricbeat" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metricbeat-deployment-modules + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.metricbeat }} + selector: + matchLabels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.metricbeat.node_selector_key }}: {{ .Values.labels.metricbeat.node_selector_value }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + runAsUser: 0 +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/etc/metricbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KUBE_STATE_METRICS_HOST + value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KUBE_STATE_METRICS_PORT + value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: metricbeat-etc + mountPath: /etc/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: deployment_kubernetes.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/mysql.yml + subPath: mysql.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/rabbitmq.yml + subPath: rabbitmq.yml + readOnly: true + volumes: + - name: metricbeat-etc + configMap: + name: metricbeat-etc + defaultMode: 0444 +{{- end }} diff --git a/elastic-metricbeat/templates/job-image-repo-sync.yaml b/elastic-metricbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..97cd86b1a --- /dev/null +++ b/elastic-metricbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml new file mode 100644 index 000000000..e10bc938e --- /dev/null +++ b/elastic-metricbeat/values.yaml @@ -0,0 +1,243 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for metricbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + metricbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat:6.3.1 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: metricbeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - metricbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + metricbeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + metricbeat: + setup: + dashboards: + enabled: true + retry: + enabled: true + interval: 5 + metricbeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + modules: + mysql: + - module: mysql + metricsets: ["status"] + hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"] + username: root + password: password + rabbitmq: + - module: rabbitmq + metricsets: ["node", "queue", "connection"] + enabled: true + period: 30s + hosts: ["rabbitmq.openstack.svc.cluster.local:15672"] + username: rabbitmq + password: password + # docker: + # - module: docker + # metricsets: + # - "container" + # - "cpu" + # - "diskio" + # - "healthcheck" + # - "info" + # - "image" + # - "memory" + # - "network" + # hosts: ["unix:///var/run/docker.sock"] + # period: 10s + # enabled: false + # labels.dedot: true + system: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + - core + - diskio + - socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + daemonset_kubernetes: + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] + deployment_kubernetes: + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - event + period: 10s + hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + kube_state_metrics: + namespace: null + hosts: + default: kube-state-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 8080 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + metricbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + metricbeat: 1 + resources: + metricbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + metricbeat: + metricbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + deployment: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-packetbeat/Chart.yaml b/elastic-packetbeat/Chart.yaml new file mode 100644 index 000000000..349f3b0d7 --- /dev/null +++ b/elastic-packetbeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Packetbeat +name: elastic-packetbeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/packetbeat +sources: + - https://github.com/elastic/beats/tree/master/packetbeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-packetbeat/requirements.yaml b/elastic-packetbeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-packetbeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-packetbeat/templates/configmap-etc.yaml b/elastic-packetbeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..b0811d094 --- /dev/null +++ b/elastic-packetbeat/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: packetbeat-etc +data: + packetbeat.yml: | +{{ toYaml .Values.conf.packetbeat | indent 4 }} +{{- end }} diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml new file mode 100644 index 000000000..b2f4a18af --- /dev/null +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -0,0 +1,144 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_packetbeat := .Values.pod.mounts.packetbeat.packetbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "packetbeat" }} +{{ tuple $envAll "packetbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: packetbeat +spec: +{{ tuple $envAll "packetbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "packetbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + securityContext: + runAsUser: 0 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "packetbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: packetbeat + image: {{ .Values.images.tags.packetbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.packetbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + args: + - "-c" + - "/etc/packetbeat/packetbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: packetbeat-etc + mountPath: /etc/packetbeat/packetbeat.yml + subPath: packetbeat.yml + readOnly: true +{{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: packetbeat-etc + configMap: + defaultMode: 0444 + name: packetbeat-etc +{{ if $mounts_packetbeat.volumes }}{{ toYaml $mounts_packetbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-packetbeat/templates/job-image-repo-sync.yaml b/elastic-packetbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..97cd86b1a --- /dev/null +++ b/elastic-packetbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml new file mode 100644 index 000000000..3f4b3b6f4 --- /dev/null +++ b/elastic-packetbeat/values.yaml @@ -0,0 +1,182 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for packetbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + packetbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + packetbeat: docker.elastic.co/beats/packetbeat:6.3.1 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: packetbeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - packetbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + packetbeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + packetbeat: + setup: + kibana: + host: ['${KIBANA_HOST}:${KIBANA_PORT}'] + dashboards: + enabled: true + retry: + enabled: true + interval: 5 + packetbeat: + interfaces: + device: any + protocols: + - type: dns + ports: [53] + include_authorities: true + include_additionals: true + - type: http + ports: [80, 8080, 8081, 5000, 8002] + - type: memcache + ports: [11211] + - type: mysql + ports: [3306] + - type: rabbit + ports: [5672, 15672] + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + packetbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + packetbeat: 1 + resources: + packetbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + packetbeat: + packetbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh new file mode 120000 index 000000000..69d244e73 --- /dev/null +++ b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh @@ -0,0 +1 @@ +../multinode/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/020-ingress.sh b/tools/deployment/elastic-beats/020-ingress.sh new file mode 120000 index 000000000..f04ad7ccd --- /dev/null +++ b/tools/deployment/elastic-beats/020-ingress.sh @@ -0,0 +1 @@ +../multinode/020-ingress.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/030-ceph.sh b/tools/deployment/elastic-beats/030-ceph.sh new file mode 120000 index 000000000..9f7b38835 --- /dev/null +++ b/tools/deployment/elastic-beats/030-ceph.sh @@ -0,0 +1 @@ +../multinode/030-ceph.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/035-ceph-ns-activate.sh b/tools/deployment/elastic-beats/035-ceph-ns-activate.sh new file mode 120000 index 000000000..f6c0f5f2e --- /dev/null +++ b/tools/deployment/elastic-beats/035-ceph-ns-activate.sh @@ -0,0 +1 @@ +../multinode/035-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/040-ldap.sh b/tools/deployment/elastic-beats/040-ldap.sh new file mode 120000 index 000000000..4ed4b9d4b --- /dev/null +++ b/tools/deployment/elastic-beats/040-ldap.sh @@ -0,0 +1 @@ +../common/040-ldap.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh new file mode 100755 index 000000000..a0a227394 --- /dev/null +++ b/tools/deployment/elastic-beats/050-elasticsearch.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elasticsearch + +#NOTE: Deploy command +tee /tmp/elasticsearch.yaml << EOF +monitoring: + prometheus: + enabled: true +pod: + replicas: + data: 1 + +EOF +helm upgrade --install elasticsearch ./elasticsearch \ + --namespace=osh-infra \ + --values=/tmp/elasticsearch.yaml + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status elasticsearch diff --git a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh b/tools/deployment/elastic-beats/060-elastic-metricbeat.sh new file mode 100755 index 000000000..1fe6cda39 --- /dev/null +++ b/tools/deployment/elastic-beats/060-elastic-metricbeat.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-metricbeat + +#NOTE: Deploy command +helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ + --namespace=kube-system \ + --set endpoints.kube_state_metrics.namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-metricbeat diff --git a/tools/deployment/elastic-beats/070-kube-state-metrics.sh b/tools/deployment/elastic-beats/070-kube-state-metrics.sh new file mode 120000 index 000000000..2a18ebb8b --- /dev/null +++ b/tools/deployment/elastic-beats/070-kube-state-metrics.sh @@ -0,0 +1 @@ +../common/070-kube-state-metrics.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/080-elastic-filebeat.sh b/tools/deployment/elastic-beats/080-elastic-filebeat.sh new file mode 100755 index 000000000..cb36c71c2 --- /dev/null +++ b/tools/deployment/elastic-beats/080-elastic-filebeat.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-filebeat + +#NOTE: Deploy command +helm upgrade --install elastic-filebeat ./elastic-filebeat \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-filebeat diff --git a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh b/tools/deployment/elastic-beats/090-elastic-packetbeat.sh new file mode 100755 index 000000000..0b596ef40 --- /dev/null +++ b/tools/deployment/elastic-beats/090-elastic-packetbeat.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-packetbeat + +#NOTE: Deploy command +helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra \ + --set endpoints.kibana.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-packetbeat diff --git a/tools/deployment/elastic-beats/100-elastic-apm-server.sh b/tools/deployment/elastic-beats/100-elastic-apm-server.sh new file mode 100755 index 000000000..1e4f70513 --- /dev/null +++ b/tools/deployment/elastic-beats/100-elastic-apm-server.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-apm-server + +#NOTE: Deploy command +helm upgrade --install elastic-apm-server ./elastic-apm-server \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-apm-server diff --git a/tools/deployment/elastic-beats/110-kibana.sh b/tools/deployment/elastic-beats/110-kibana.sh new file mode 120000 index 000000000..e6ed5dfb6 --- /dev/null +++ b/tools/deployment/elastic-beats/110-kibana.sh @@ -0,0 +1 @@ +../osh-infra-logging/070-kibana.sh \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 5702926cf..d37b768c4 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -208,3 +208,30 @@ - ./tools/deployment/keystone-auth/060-mariadb.sh - ./tools/deployment/keystone-auth/070-keystone.sh - ./tools/deployment/keystone-auth/080-check.sh + +- job: + name: openstack-helm-infra-elastic-beats + parent: openstack-helm-infra-functional + nodeset: openstack-helm-five-node-ubuntu + timeout: 7200 + pre-run: + - playbooks/osh-infra-upgrade-host.yaml + - playbooks/osh-infra-deploy-docker.yaml + - playbooks/osh-infra-build.yaml + - playbooks/osh-infra-deploy-k8s.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + vars: + gate_scripts: + - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh + - ./tools/deployment/elastic-beats/020-ingress.sh + - ./tools/deployment/elastic-beats/030-ceph.sh + - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh + - ./tools/deployment/elastic-beats/040-ldap.sh + - ./tools/deployment/elastic-beats/050-elasticsearch.sh + - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh + - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh + - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh + - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh + - ./tools/deployment/elastic-beats/110-kibana.sh diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 356b6814e..1e5ff7739 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -40,3 +40,4 @@ - openstack-helm-infra-five-fedora #NOTE(srwilkers): Make centos job experimental until issues resolved - openstack-helm-infra-five-centos + - openstack-helm-infra-elastic-beats