diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml index 2e2fc1232..29e448a3f 100644 --- a/elastic-filebeat/templates/configmap-etc.yaml +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -23,5 +23,7 @@ metadata: name: filebeat-etc data: filebeat.yml: | -{{ toYaml .Values.conf | indent 4 }} +{{ toYaml .Values.conf.filebeat | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} {{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml index 26f524136..e146c23bb 100644 --- a/elastic-filebeat/templates/daemonset.yaml +++ b/elastic-filebeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -105,10 +106,18 @@ spec: - name: filebeat containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: ELASTICSEARCH_HOST value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -133,6 +142,10 @@ spec: mountPath: /usr/share/filebeat/filebeat.yml readOnly: true subPath: filebeat.yml + - name: filebeat-etc + mountPath: /usr/share/filebeat/modules.d/system.yml + subPath: system.yml + readOnly: true {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} volumes: - name: pod-tmp diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml index 92289fd1d..c0c1b76f4 100644 --- a/elastic-filebeat/values.yaml +++ b/elastic-filebeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - filebeat: docker.elastic.co/beats/filebeat:6.2.3 + filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,82 +53,106 @@ dependencies: service: local_image_registry static: filebeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal service: local_image_registry conf: - setup: - dashboards: - enabled: true - path: - logs: /var/log/ - output: - elasticsearch: - hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] - username: "${ELASTICSEARCH_USERNAME}" - password: "${ELASTICSEARCH_PASSWORD}" filebeat: - config: - prospectors: - path: ${path.config}/prospectors.d/*.yml - reload: - enabled: false - modules: - path: ${path.config}/modules.d/*.yml - reload: - enabled: false - autodiscover: - providers: - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: kube-system - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: ceph - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - - type: kubernetes - templates: - - condition: - equals: - kubernetes.namespace: openstack - config: - - type: docker - containers.ids: - - "${data.kubernetes.container.id}" - exclude_lines: ["^\\s+[\\-`('.|_]"] - prospectors: - - type: docker - containers.ids: - - "*" - multiline: - pattern: '^Traceback' - match: after - negate: true - processors: - - add_kubernetes_metadata: - in_cluster: true - - drop_event: - when: - equals: - kubernetes: - container: - name: "filebeat" + setup: + dashboards: + enabled: true + index: "filebeat-*" + retry: + enabled: true + interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: osh-infra + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + modules: + system: + - module: system + syslog: + enabled: true + var.paths: ["/var/log/syslog*"] + fields: + host: + name: "${NODE_NAME}" + auth: + enabled: true + var.paths: ["/var/log/auth.log"] + fields: + host: + name: "${NODE_NAME}" endpoints: cluster_domain_suffix: cluster.local @@ -165,6 +189,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 filebeat: namespace: null name: filebeat diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml index 51595e2f3..9f47a8393 100644 --- a/elastic-metricbeat/templates/daemonset-node-metrics.yaml +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -48,8 +48,10 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges + - events verbs: - get - list @@ -104,7 +106,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" - "-system.hostfs=/hostfs" env: @@ -112,6 +114,10 @@ spec: value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: ELASTICSEARCH_PORT value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -130,7 +136,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml index 4e1c60236..7ffadb674 100644 --- a/elastic-metricbeat/templates/deployment-modules.yaml +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -19,9 +19,54 @@ limitations under the License. {{- $esUserSecret := .Values.secrets.elasticsearch.user }} -{{- $serviceAccountName := "metricbeat" }} +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }} {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + - events + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -57,7 +102,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} args: - "-c" - - "/etc/metricbeat.yml" + - "/usr/share/metricbeat/metricbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -68,6 +113,10 @@ spec: value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} - name: KUBE_STATE_METRICS_PORT value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -86,7 +135,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: metricbeat-etc - mountPath: /etc/metricbeat.yml + mountPath: /usr/share/metricbeat/metricbeat.yml subPath: metricbeat.yml readOnly: true - name: metricbeat-etc diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml index c8a4a3e3c..636636268 100644 --- a/elastic-metricbeat/values.yaml +++ b/elastic-metricbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - metricbeat: docker.elastic.co/beats/metricbeat:6.3.1 + metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -53,7 +53,9 @@ dependencies: service: local_image_registry static: metricbeat: - services: null + services: + - endpoint: internal + service: kibana image_repo_sync: services: - endpoint: internal @@ -64,9 +66,14 @@ conf: setup: dashboards: enabled: true + index: metricbeat-* retry: enabled: true interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" metricbeat: config: modules: @@ -79,35 +86,20 @@ conf: username: ${ELASTICSEARCH_USERNAME} password: ${ELASTICSEARCH_PASSWORD} modules: - mysql: - - module: mysql - metricsets: ["status"] - hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"] - username: root - password: password - rabbitmq: - - module: rabbitmq - metricsets: ["node", "queue", "connection"] + docker: + - module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + - "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s enabled: true - period: 30s - hosts: ["rabbitmq.openstack.svc.cluster.local:15672"] - username: rabbitmq - password: password - # docker: - # - module: docker - # metricsets: - # - "container" - # - "cpu" - # - "diskio" - # - "healthcheck" - # - "info" - # - "image" - # - "memory" - # - "network" - # hosts: ["unix:///var/run/docker.sock"] - # period: 10s - # enabled: false - # labels.dedot: true system: - module: system period: 10s @@ -121,18 +113,15 @@ conf: - core - diskio - socket + - filesystem + - fsstat processes: ['.*'] + cpu.metrics: ["percentages"] + core.metrics: ["percentages"] process.include_top_n: by_cpu: 5 by_memory: 5 - - module: system - period: 1m - metricsets: - - filesystem - - fsstat - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' + enabled: true daemonset_kubernetes: - module: kubernetes metricsets: @@ -143,6 +132,9 @@ conf: - volume period: 10s hosts: ["localhost:10255"] + add_metadata: true + in_cluster: true + enabled: true deployment_kubernetes: - module: kubernetes metricsets: @@ -154,6 +146,9 @@ conf: - event period: 10s hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + add_metadata: true + in_cluster: true + enabled: true endpoints: cluster_domain_suffix: cluster.local @@ -203,6 +198,23 @@ endpoints: port: http: default: 80 + kibana: + name: kibana + namespace: osh-infra + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 pod: affinity: diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml index 6db4cdac4..7978b652c 100644 --- a/elastic-packetbeat/templates/daemonset.yaml +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -48,6 +48,7 @@ rules: - nodes - pods - services + - endpoints - replicationcontrollers - limitranges verbs: @@ -102,7 +103,7 @@ spec: - NET_ADMIN args: - "-c" - - "/etc/packetbeat/packetbeat.yml" + - "/usr/share/packetbeat/packetbeat.yml" - "-e" env: - name: ELASTICSEARCH_HOST @@ -131,7 +132,7 @@ spec: - name: pod-tmp mountPath: /tmp - name: packetbeat-etc - mountPath: /etc/packetbeat/packetbeat.yml + mountPath: /usr/share/packetbeat/packetbeat.yml subPath: packetbeat.yml readOnly: true {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml index 78955e429..2b163ed8f 100644 --- a/elastic-packetbeat/values.yaml +++ b/elastic-packetbeat/values.yaml @@ -28,7 +28,7 @@ labels: images: tags: - packetbeat: docker.elastic.co/beats/packetbeat:6.3.1 + packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 pull_policy: IfNotPresent @@ -63,28 +63,30 @@ conf: packetbeat: setup: kibana: - host: ['${KIBANA_HOST}:${KIBANA_PORT}'] + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" dashboards: enabled: true + index: "packetbeat-*" retry: enabled: true interval: 5 packetbeat: + flows: + timeout: 30s + period: 10s interfaces: device: any protocols: + - type: dhcpv4 + ports: [67, 68] - type: dns ports: [53] include_authorities: true include_additionals: true - type: http - ports: [80, 8080, 8081, 5000, 8002] - - type: memcache - ports: [11211] - - type: mysql - ports: [3306] - - type: rabbit - ports: [5672, 15672] + ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134] output: elasticsearch: hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] diff --git a/tools/deployment/elastic-beats/005-deploy-k8s.sh b/tools/deployment/elastic-beats/005-deploy-k8s.sh new file mode 120000 index 000000000..257a39f7a --- /dev/null +++ b/tools/deployment/elastic-beats/005-deploy-k8s.sh @@ -0,0 +1 @@ +../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh deleted file mode 120000 index 69d244e73..000000000 --- a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh index a0a227394..0313c64e7 100755 --- a/tools/deployment/elastic-beats/050-elasticsearch.sh +++ b/tools/deployment/elastic-beats/050-elasticsearch.sh @@ -21,12 +21,39 @@ make elasticsearch #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF +manifests: + cron_curator: false + configmap_bin_curator: false + configmap_etc_curator: false +images: + tags: + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115 +storage: + data: + requests: + storage: 20Gi + master: + requests: + storage: 5Gi +jobs: + verify_repositories: + cron: "*/10 * * * *" monitoring: prometheus: - enabled: true + enabled: false pod: replicas: + client: 1 data: 1 + master: 2 +conf: + elasticsearch: + config: + xpack: + security: + enabled: false + ilm: + enabled: false EOF helm upgrade --install elasticsearch ./elasticsearch \ diff --git a/tools/deployment/elastic-beats/060-kibana.sh b/tools/deployment/elastic-beats/060-kibana.sh new file mode 100755 index 000000000..ac955075b --- /dev/null +++ b/tools/deployment/elastic-beats/060-kibana.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make kibana + +: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} + +tee /tmp/kibana.yaml << EOF +images: + tags: + kibana: docker.elastic.co/kibana/kibana:7.1.0 +conf: + kibana: + xpack: + security: + enabled: false + spaces: + enabled: false + apm: + enabled: false + graph: + enabled: false + ml: + enabled: false + monitoring: + enabled: false + reporting: + enabled: false + canvas: + enabled: false +EOF + +#NOTE: Deploy command +helm upgrade --install kibana ./kibana \ + --namespace=osh-infra \ + --values=/tmp/kibana.yaml + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh osh-infra + +#NOTE: Validate Deployment info +helm status kibana diff --git a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/060-elastic-metricbeat.sh rename to tools/deployment/elastic-beats/080-elastic-metricbeat.sh index 1fe6cda39..cf3c73400 100755 --- a/tools/deployment/elastic-beats/060-elastic-metricbeat.sh +++ b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-metricbeat +tee /tmp/metricbeat.yaml << EOF +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat:7.1.0 +conf: + metricbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ --namespace=kube-system \ - --set endpoints.kube_state_metrics.namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/metricbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/080-elastic-filebeat.sh b/tools/deployment/elastic-beats/090-elastic-filebeat.sh similarity index 76% rename from tools/deployment/elastic-beats/080-elastic-filebeat.sh rename to tools/deployment/elastic-beats/090-elastic-filebeat.sh index cb36c71c2..f73848095 100755 --- a/tools/deployment/elastic-beats/080-elastic-filebeat.sh +++ b/tools/deployment/elastic-beats/090-elastic-filebeat.sh @@ -19,10 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-filebeat +tee /tmp/filebeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/filebeat:7.1.0 +conf: + filebeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-filebeat ./elastic-filebeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra + --values=/tmp/filebeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/100-elastic-apm-server.sh b/tools/deployment/elastic-beats/100-elastic-apm-server.sh deleted file mode 100755 index 1e4f70513..000000000 --- a/tools/deployment/elastic-beats/100-elastic-apm-server.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-apm-server - -#NOTE: Deploy command -helm upgrade --install elastic-apm-server ./elastic-apm-server \ - --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system - -#NOTE: Validate Deployment info -helm status elastic-apm-server diff --git a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh similarity index 76% rename from tools/deployment/elastic-beats/090-elastic-packetbeat.sh rename to tools/deployment/elastic-beats/100-elastic-packetbeat.sh index 0b596ef40..c0fc08672 100755 --- a/tools/deployment/elastic-beats/090-elastic-packetbeat.sh +++ b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh @@ -19,11 +19,26 @@ set -xe #NOTE: Lint and package chart make elastic-packetbeat +tee /tmp/packetbeat.yaml << EOF +images: + tags: + filebeat: docker.elastic.co/beats/packetbeat:7.1.0 +conf: + packetbeat: + setup: + ilm: + enabled: false +endpoints: + elasticsearch: + namespace: osh-infra + kibana: + namespace: osh-infra +EOF + #NOTE: Deploy command helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ --namespace=kube-system \ - --set endpoints.elasticsearch.namespace=osh-infra \ - --set endpoints.kibana.namespace=osh-infra + --values=/tmp/packetbeat.yaml #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/110-kibana.sh b/tools/deployment/elastic-beats/110-kibana.sh deleted file mode 120000 index 03852eb06..000000000 --- a/tools/deployment/elastic-beats/110-kibana.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/075-kibana.sh \ No newline at end of file diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 8385a59e8..1026c9e00 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -348,29 +348,24 @@ - job: name: openstack-helm-infra-elastic-beats parent: openstack-helm-infra-functional - nodeset: openstack-helm-five-node-ubuntu timeout: 7200 - pre-run: - - playbooks/osh-infra-upgrade-host.yaml - - playbooks/osh-infra-deploy-docker.yaml - - playbooks/osh-infra-build.yaml - - playbooks/osh-infra-deploy-k8s.yaml + pre-run: playbooks/osh-infra-upgrade-host.yaml run: playbooks/osh-infra-gate-runner.yaml post-run: playbooks/osh-infra-collect-logs.yaml + nodeset: openstack-helm-single-node vars: gate_scripts: - - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh + - ./tools/deployment/elastic-beats/005-deploy-k8s.sh - ./tools/deployment/elastic-beats/020-ingress.sh - ./tools/deployment/elastic-beats/030-ceph.sh - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh - ./tools/deployment/elastic-beats/040-ldap.sh - ./tools/deployment/elastic-beats/050-elasticsearch.sh - - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/060-kibana.sh - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh - - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh - - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh - - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh - - ./tools/deployment/elastic-beats/110-kibana.sh + - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh + - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh + - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh - job: name: openstack-helm-infra-armada-deploy diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index ec3e29410..5b95b3ec6 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -57,15 +57,15 @@ # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords - # experimental: - # jobs: + experimental: + jobs: # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved #- openstack-helm-infra-five-fedora # NOTE(srwilkers): Disable centos experimental jobs until issues resolved #- openstack-helm-infra-five-centos # - openstack-helm-infra-five-ubuntu + - openstack-helm-infra-elastic-beats # - openstack-helm-infra-tenant-ceph - # - openstack-helm-infra-elastic-beats # - openstack-helm-infra-armada-deploy # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords