diff --git a/elastic-apm-server/Chart.yaml b/elastic-apm-server/Chart.yaml new file mode 100644 index 000000000..6eac8ade2 --- /dev/null +++ b/elastic-apm-server/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic APM Server +name: elastic-apm-server +version: 0.1.0 +home: https://www.elastic.co/guide/en/apm/get-started/current/index.html +sources: + - https://github.com/elastic/apm-server + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-apm-server/requirements.yaml b/elastic-apm-server/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-apm-server/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-apm-server/templates/configmap-bin.yaml b/elastic-apm-server/templates/configmap-bin.yaml new file mode 100644 index 000000000..d26f95872 --- /dev/null +++ b/elastic-apm-server/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/configmap-etc.yaml b/elastic-apm-server/templates/configmap-etc.yaml new file mode 100644 index 000000000..6d584d02c --- /dev/null +++ b/elastic-apm-server/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-etc +data: + apm-server.yml: | +{{ toYaml .Values.conf.apm_server | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/deployment.yaml b/elastic-apm-server/templates/deployment.yaml new file mode 100644 index 000000000..19ff4fffc --- /dev/null +++ b/elastic-apm-server/templates/deployment.yaml @@ -0,0 +1,123 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_elastic_apm_server := .Values.pod.mounts.elastic_apm_server.elastic_apm_server }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elastic-apm-server" }} +{{ tuple $envAll "elastic-apm-server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +- apiGroups: [""] + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: elastic-apm-server +spec: +{{ tuple $envAll "elastic-apm-server" | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + replicas: {{ .Values.pod.replicas.elastic_apm_server }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.elastic_apm_server.node_selector_key }}: {{ .Values.labels.elastic_apm_server.node_selector_value }} + initContainers: +{{ tuple $envAll "elastic_apm_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: elastic-apm-server + image: {{ .Values.images.tags.elastic_apm_server }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.elastic_apm_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/usr/share/apm-server/apm-server.yml" + - "-e" + ports: + - name: server + containerPort: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: APM_SERVER_HOST + value: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: APM_SERVER_PORT + value: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: elastic-apm-server-etc + mountPath: /usr/share/apm-server/apm-server.yml + readOnly: true + subPath: apm-server.yml + - name: data + mountPath: /usr/share/apm-server/data +{{ if $mounts_elastic_apm_server.volumeMounts }}{{ toYaml $mounts_elastic_apm_server.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: elastic-apm-server-etc + configMap: + name: elastic-apm-server-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/elastic-apm-server + type: DirectoryOrCreate +{{ if $mounts_elastic_apm_server.volumes }}{{ toYaml $mounts_elastic_apm_server.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-apm-server/templates/job-image-repo-sync.yaml b/elastic-apm-server/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..cb701d04c --- /dev/null +++ b/elastic-apm-server/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-apm-server/templates/secret-elasticsearch-creds.yaml b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-apm-server/templates/service.yaml b/elastic-apm-server/templates/service.yaml new file mode 100644 index 000000000..d4aed6554 --- /dev/null +++ b/elastic-apm-server/templates/service.yaml @@ -0,0 +1,34 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: server + port: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + nodePort: {{ .Values.network.elastic_apm_server.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + type: NodePort + {{ end }} diff --git a/elastic-apm-server/values.yaml b/elastic-apm-server/values.yaml new file mode 100644 index 000000000..bc1f94b05 --- /dev/null +++ b/elastic-apm-server/values.yaml @@ -0,0 +1,166 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elastic-apm-server +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + elastic_apm_server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + elastic_apm_server: docker.elastic.co/apm/apm-server:6.2.3 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: elastic-apm-server-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - elastic-apm-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + elastic_apm_server: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + apm_server: + setup: + dashboards: + enabled: true + host: ['${APM_SERVER_HOST}:${APM_SERVER_PORT}'] + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + elastic_apm_server: + namespace: null + name: apm-server + hosts: + default: apm-server + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + server: + default: 8200 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + elastic_apm_server: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + elastic_apm_server: 1 + resources: + elastic_apm_server: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + elastic_apm_server: + elastic_apm_server: + +network: + elastic_apm_server: + node_port: + enabled: false + port: 30200 + +manifests: + configmap_bin: true + configmap_etc: true + deployment: true + service: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-filebeat/Chart.yaml b/elastic-filebeat/Chart.yaml new file mode 100644 index 000000000..3543e920c --- /dev/null +++ b/elastic-filebeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Filebeat +name: elastic-filebeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/filebeat +sources: + - https://github.com/elastic/beats/tree/master/filebeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-filebeat/requirements.yaml b/elastic-filebeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-filebeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-filebeat/templates/configmap-bin.yaml b/elastic-filebeat/templates/configmap-bin.yaml new file mode 100644 index 000000000..149f7fff4 --- /dev/null +++ b/elastic-filebeat/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..2e2fc1232 --- /dev/null +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-etc +data: + filebeat.yml: | +{{ toYaml .Values.conf | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml new file mode 100644 index 000000000..655307e94 --- /dev/null +++ b/elastic-filebeat/templates/daemonset.yaml @@ -0,0 +1,150 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_filebeat := .Values.pod.mounts.filebeat.filebeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "filebeat" }} +{{ tuple $envAll "filebeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat +spec: +{{ tuple $envAll "filebeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "filebeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.filebeat.node_selector_key }}: {{ .Values.labels.filebeat.node_selector_value }} + initContainers: +{{ tuple $envAll "filebeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: filebeat + image: {{ .Values.images.tags.filebeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.filebeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-e" + ports: + - name: filebeat + containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: data + mountPath: /usr/share/filebeat/data + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: filebeat-etc + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true + subPath: filebeat.yml +{{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: filebeat-etc + configMap: + name: filebeat-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/filebeat + type: DirectoryOrCreate +{{ if $mounts_filebeat.volumes }}{{ toYaml $mounts_filebeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-filebeat/templates/job-image-repo-sync.yaml b/elastic-filebeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..cb701d04c --- /dev/null +++ b/elastic-filebeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-filebeat/templates/secret-elasticsearch-creds.yaml b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml new file mode 100644 index 000000000..00cb6b406 --- /dev/null +++ b/elastic-filebeat/values.yaml @@ -0,0 +1,216 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for filebeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + filebeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + filebeat: docker.elastic.co/beats/filebeat:6.2.3 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: filebeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - filebeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + filebeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + setup: + dashboards: + enabled: true + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + prospectors: + path: ${path.config}/prospectors.d/*.yml + reload: + enabled: false + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: false + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + prospectors: + - type: docker + containers.ids: + - "*" + multiline: + pattern: '^Traceback' + match: after + negate: true + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + filebeat: + namespace: null + name: filebeat + hosts: + default: filebeat + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 5066 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + filebeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + filebeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + filebeat: + filebeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-metricbeat/Chart.yaml b/elastic-metricbeat/Chart.yaml new file mode 100644 index 000000000..0f4d40c94 --- /dev/null +++ b/elastic-metricbeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Metricbeat +name: elastic-metricbeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/metricbeat +sources: + - https://github.com/elastic/beats/tree/master/metricbeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-metricbeat/requirements.yaml b/elastic-metricbeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-metricbeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-metricbeat/templates/configmap-etc.yaml b/elastic-metricbeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..bada69fcc --- /dev/null +++ b/elastic-metricbeat/templates/configmap-etc.yaml @@ -0,0 +1,37 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-etc +data: + metricbeat.yml: | +{{ toYaml .Values.conf.metricbeat | indent 4 }} + rabbitmq.yml: | +{{ toYaml .Values.conf.modules.rabbitmq | indent 4 }} + mysql.yml: | +{{ toYaml .Values.conf.modules.mysql | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} + daemonset_kubernetes.yml: | +{{ toYaml .Values.conf.modules.daemonset_kubernetes | indent 4 }} + deployment_kubernetes.yml: | +{{ toYaml .Values.conf.modules.deployment_kubernetes | indent 4 }} +{{- end }} diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml new file mode 100644 index 000000000..64c6e091b --- /dev/null +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -0,0 +1,165 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_metricbeat := .Values.pod.mounts.metricbeat.metricbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: metricbeat-node-modules +spec: +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + privileged: true + runAsUser: 0 + image: {{ .Values.images.tags.metricbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/etc/metricbeat.yml" + - "-e" + - "-system.hostfs=/hostfs" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: metricbeat-etc + mountPath: /etc/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/system.yml + subPath: system.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: daemonset_kubernetes.yml + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true +{{ if $mounts_metricbeat.volumeMounts }}{{ toYaml $mounts_metricbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: metricbeat-etc + configMap: + defaultMode: 0444 + name: metricbeat-etc + - name: data + emptyDir: {} +{{ if $mounts_metricbeat.volumes }}{{ toYaml $mounts_metricbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml new file mode 100644 index 000000000..e4162019b --- /dev/null +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -0,0 +1,106 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "metricbeat" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metricbeat-deployment-modules + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.metricbeat }} + selector: + matchLabels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.metricbeat.node_selector_key }}: {{ .Values.labels.metricbeat.node_selector_value }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + runAsUser: 0 +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/etc/metricbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KUBE_STATE_METRICS_HOST + value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KUBE_STATE_METRICS_PORT + value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: metricbeat-etc + mountPath: /etc/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: deployment_kubernetes.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/mysql.yml + subPath: mysql.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/rabbitmq.yml + subPath: rabbitmq.yml + readOnly: true + volumes: + - name: metricbeat-etc + configMap: + name: metricbeat-etc + defaultMode: 0444 +{{- end }} diff --git a/elastic-metricbeat/templates/job-image-repo-sync.yaml b/elastic-metricbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..97cd86b1a --- /dev/null +++ b/elastic-metricbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml new file mode 100644 index 000000000..e10bc938e --- /dev/null +++ b/elastic-metricbeat/values.yaml @@ -0,0 +1,243 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for metricbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + metricbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat:6.3.1 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: metricbeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - metricbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + metricbeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + metricbeat: + setup: + dashboards: + enabled: true + retry: + enabled: true + interval: 5 + metricbeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + modules: + mysql: + - module: mysql + metricsets: ["status"] + hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"] + username: root + password: password + rabbitmq: + - module: rabbitmq + metricsets: ["node", "queue", "connection"] + enabled: true + period: 30s + hosts: ["rabbitmq.openstack.svc.cluster.local:15672"] + username: rabbitmq + password: password + # docker: + # - module: docker + # metricsets: + # - "container" + # - "cpu" + # - "diskio" + # - "healthcheck" + # - "info" + # - "image" + # - "memory" + # - "network" + # hosts: ["unix:///var/run/docker.sock"] + # period: 10s + # enabled: false + # labels.dedot: true + system: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + - core + - diskio + - socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + - module: system + period: 1m + metricsets: + - filesystem + - fsstat + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + daemonset_kubernetes: + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] + deployment_kubernetes: + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - event + period: 10s + hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + kube_state_metrics: + namespace: null + hosts: + default: kube-state-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 8080 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + metricbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + metricbeat: 1 + resources: + metricbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + metricbeat: + metricbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + deployment: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/elastic-packetbeat/Chart.yaml b/elastic-packetbeat/Chart.yaml new file mode 100644 index 000000000..349f3b0d7 --- /dev/null +++ b/elastic-packetbeat/Chart.yaml @@ -0,0 +1,24 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: OpenStack-Helm Elastic Packetbeat +name: elastic-packetbeat +version: 0.1.0 +home: https://www.elastic.co/products/beats/packetbeat +sources: + - https://github.com/elastic/beats/tree/master/packetbeat + - https://git.openstack.org/cgit/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors diff --git a/elastic-packetbeat/requirements.yaml b/elastic-packetbeat/requirements.yaml new file mode 100644 index 000000000..a93ba00c4 --- /dev/null +++ b/elastic-packetbeat/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: helm-toolkit + repository: http://localhost:8879/charts/ + version: 0.1.0 diff --git a/elastic-packetbeat/templates/configmap-etc.yaml b/elastic-packetbeat/templates/configmap-etc.yaml new file mode 100644 index 000000000..b0811d094 --- /dev/null +++ b/elastic-packetbeat/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: packetbeat-etc +data: + packetbeat.yml: | +{{ toYaml .Values.conf.packetbeat | indent 4 }} +{{- end }} diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml new file mode 100644 index 000000000..b2f4a18af --- /dev/null +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -0,0 +1,144 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_packetbeat := .Values.pod.mounts.packetbeat.packetbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "packetbeat" }} +{{ tuple $envAll "packetbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - watch +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: packetbeat +spec: +{{ tuple $envAll "packetbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "packetbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + securityContext: + runAsUser: 0 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "packetbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: packetbeat + image: {{ .Values.images.tags.packetbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.packetbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + args: + - "-c" + - "/etc/packetbeat/packetbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: packetbeat-etc + mountPath: /etc/packetbeat/packetbeat.yml + subPath: packetbeat.yml + readOnly: true +{{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: packetbeat-etc + configMap: + defaultMode: 0444 + name: packetbeat-etc +{{ if $mounts_packetbeat.volumes }}{{ toYaml $mounts_packetbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-packetbeat/templates/job-image-repo-sync.yaml b/elastic-packetbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 000000000..97cd86b1a --- /dev/null +++ b/elastic-packetbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,20 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 000000000..0ea91703f --- /dev/null +++ b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml new file mode 100644 index 000000000..3f4b3b6f4 --- /dev/null +++ b/elastic-packetbeat/values.yaml @@ -0,0 +1,182 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for packetbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +release_group: null + +labels: + packetbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + packetbeat: docker.elastic.co/beats/packetbeat:6.3.1 + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 + image_repo_sync: docker.io/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: packetbeat-elasticsearch-user + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - packetbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + packetbeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + packetbeat: + setup: + kibana: + host: ['${KIBANA_HOST}:${KIBANA_PORT}'] + dashboards: + enabled: true + retry: + enabled: true + interval: 5 + packetbeat: + interfaces: + device: any + protocols: + - type: dns + ports: [53] + include_authorities: true + include_additionals: true + - type: http + ports: [80, 8080, 8081, 5000, 8002] + - type: memcache + ports: [11211] + - type: mysql + ports: [3306] + - type: rabbit + ports: [5672, 15672] + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + packetbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + packetbeat: 1 + resources: + packetbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + packetbeat: + packetbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true diff --git a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh new file mode 120000 index 000000000..69d244e73 --- /dev/null +++ b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh @@ -0,0 +1 @@ +../multinode/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/020-ingress.sh b/tools/deployment/elastic-beats/020-ingress.sh new file mode 120000 index 000000000..f04ad7ccd --- /dev/null +++ b/tools/deployment/elastic-beats/020-ingress.sh @@ -0,0 +1 @@ +../multinode/020-ingress.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/030-ceph.sh b/tools/deployment/elastic-beats/030-ceph.sh new file mode 120000 index 000000000..9f7b38835 --- /dev/null +++ b/tools/deployment/elastic-beats/030-ceph.sh @@ -0,0 +1 @@ +../multinode/030-ceph.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/035-ceph-ns-activate.sh b/tools/deployment/elastic-beats/035-ceph-ns-activate.sh new file mode 120000 index 000000000..f6c0f5f2e --- /dev/null +++ b/tools/deployment/elastic-beats/035-ceph-ns-activate.sh @@ -0,0 +1 @@ +../multinode/035-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/040-ldap.sh b/tools/deployment/elastic-beats/040-ldap.sh new file mode 120000 index 000000000..4ed4b9d4b --- /dev/null +++ b/tools/deployment/elastic-beats/040-ldap.sh @@ -0,0 +1 @@ +../common/040-ldap.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh new file mode 100755 index 000000000..a0a227394 --- /dev/null +++ b/tools/deployment/elastic-beats/050-elasticsearch.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elasticsearch + +#NOTE: Deploy command +tee /tmp/elasticsearch.yaml << EOF +monitoring: + prometheus: + enabled: true +pod: + replicas: + data: 1 + +EOF +helm upgrade --install elasticsearch ./elasticsearch \ + --namespace=osh-infra \ + --values=/tmp/elasticsearch.yaml + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status elasticsearch diff --git a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh b/tools/deployment/elastic-beats/060-elastic-metricbeat.sh new file mode 100755 index 000000000..1fe6cda39 --- /dev/null +++ b/tools/deployment/elastic-beats/060-elastic-metricbeat.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-metricbeat + +#NOTE: Deploy command +helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ + --namespace=kube-system \ + --set endpoints.kube_state_metrics.namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-metricbeat diff --git a/tools/deployment/elastic-beats/070-kube-state-metrics.sh b/tools/deployment/elastic-beats/070-kube-state-metrics.sh new file mode 120000 index 000000000..2a18ebb8b --- /dev/null +++ b/tools/deployment/elastic-beats/070-kube-state-metrics.sh @@ -0,0 +1 @@ +../common/070-kube-state-metrics.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/080-elastic-filebeat.sh b/tools/deployment/elastic-beats/080-elastic-filebeat.sh new file mode 100755 index 000000000..cb36c71c2 --- /dev/null +++ b/tools/deployment/elastic-beats/080-elastic-filebeat.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-filebeat + +#NOTE: Deploy command +helm upgrade --install elastic-filebeat ./elastic-filebeat \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-filebeat diff --git a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh b/tools/deployment/elastic-beats/090-elastic-packetbeat.sh new file mode 100755 index 000000000..0b596ef40 --- /dev/null +++ b/tools/deployment/elastic-beats/090-elastic-packetbeat.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-packetbeat + +#NOTE: Deploy command +helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra \ + --set endpoints.kibana.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-packetbeat diff --git a/tools/deployment/elastic-beats/100-elastic-apm-server.sh b/tools/deployment/elastic-beats/100-elastic-apm-server.sh new file mode 100755 index 000000000..1e4f70513 --- /dev/null +++ b/tools/deployment/elastic-beats/100-elastic-apm-server.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make elastic-apm-server + +#NOTE: Deploy command +helm upgrade --install elastic-apm-server ./elastic-apm-server \ + --namespace=kube-system \ + --set endpoints.elasticsearch.namespace=osh-infra + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Validate Deployment info +helm status elastic-apm-server diff --git a/tools/deployment/elastic-beats/110-kibana.sh b/tools/deployment/elastic-beats/110-kibana.sh new file mode 120000 index 000000000..e6ed5dfb6 --- /dev/null +++ b/tools/deployment/elastic-beats/110-kibana.sh @@ -0,0 +1 @@ +../osh-infra-logging/070-kibana.sh \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 5702926cf..d37b768c4 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -208,3 +208,30 @@ - ./tools/deployment/keystone-auth/060-mariadb.sh - ./tools/deployment/keystone-auth/070-keystone.sh - ./tools/deployment/keystone-auth/080-check.sh + +- job: + name: openstack-helm-infra-elastic-beats + parent: openstack-helm-infra-functional + nodeset: openstack-helm-five-node-ubuntu + timeout: 7200 + pre-run: + - playbooks/osh-infra-upgrade-host.yaml + - playbooks/osh-infra-deploy-docker.yaml + - playbooks/osh-infra-build.yaml + - playbooks/osh-infra-deploy-k8s.yaml + run: playbooks/osh-infra-gate-runner.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + vars: + gate_scripts: + - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh + - ./tools/deployment/elastic-beats/020-ingress.sh + - ./tools/deployment/elastic-beats/030-ceph.sh + - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh + - ./tools/deployment/elastic-beats/040-ldap.sh + - ./tools/deployment/elastic-beats/050-elasticsearch.sh + - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh + - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh + - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh + - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh + - ./tools/deployment/elastic-beats/110-kibana.sh diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 356b6814e..1e5ff7739 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -40,3 +40,4 @@ - openstack-helm-infra-five-fedora #NOTE(srwilkers): Make centos job experimental until issues resolved - openstack-helm-infra-five-centos + - openstack-helm-infra-elastic-beats