Merge "Update Elastic Beats versions to 7.1.0"

elastic-filebeat/templates/configmap-etc.yaml

@@ -23,5 +23,7 @@ metadata:
   name: filebeat-etc
 data:
   filebeat.yml: |
-{{ toYaml .Values.conf | indent 4 }}
+{{ toYaml .Values.conf.filebeat | indent 4 }}
+  system.yml: |
+{{ toYaml .Values.conf.modules.system | indent 4 }}
 {{- end }}

elastic-filebeat/templates/daemonset.yaml

@@ -48,6 +48,7 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
     verbs:
@@ -105,10 +106,18 @@ spec:
             - name: filebeat
               containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: ELASTICSEARCH_HOST
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -133,6 +142,10 @@ spec:
               mountPath: /usr/share/filebeat/filebeat.yml
               readOnly: true
               subPath: filebeat.yml
+            - name: filebeat-etc
+              mountPath: /usr/share/filebeat/modules.d/system.yml
+              subPath: system.yml
+              readOnly: true
 {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }}
       volumes:
         - name: pod-tmp

elastic-filebeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    filebeat: docker.elastic.co/beats/filebeat:6.2.3
+    filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -53,82 +53,106 @@ dependencies:
             service: local_image_registry
   static:
     filebeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
     image_repo_sync:
       services:
         - endpoint: internal
           service: local_image_registry
 
 conf:
-  setup:
-    dashboards:
-      enabled: true
-  path:
-    logs: /var/log/
-  output:
-    elasticsearch:
-      hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
-      username: "${ELASTICSEARCH_USERNAME}"
-      password: "${ELASTICSEARCH_PASSWORD}"
   filebeat:
-    config:
+    setup:
-      prospectors:
+      dashboards:
-        path: ${path.config}/prospectors.d/*.yml
+        enabled: true
-        reload:
+        index: "filebeat-*"
-          enabled: false
+        retry:
-      modules:
+          enabled: true
-        path: ${path.config}/modules.d/*.yml
+          interval: 5
-        reload:
+      kibana:
-          enabled: false
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
-    autodiscover:
+        username: "${ELASTICSEARCH_USERNAME}"
-      providers:
+        password: "${ELASTICSEARCH_PASSWORD}"
-        - type: kubernetes
+    path:
-          templates:
+      logs: /var/log/
-            - condition:
+    output:
-                equals:
+      elasticsearch:
-                  kubernetes.namespace: kube-system
+        hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"]
-              config:
+        username: "${ELASTICSEARCH_USERNAME}"
-                - type: docker
+        password: "${ELASTICSEARCH_PASSWORD}"
-                  containers.ids:
+    filebeat:
-                    - "${data.kubernetes.container.id}"
+      config:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+        modules:
-        - type: kubernetes
+          path: ${path.config}/modules.d/*.yml
-          templates:
+          reload:
-            - condition:
+            enabled: true
-                equals:
+      autodiscover:
-                  kubernetes.namespace: ceph
+        providers:
-              config:
+          - type: kubernetes
-                - type: docker
+            templates:
-                  containers.ids:
+              - condition:
-                    - "${data.kubernetes.container.id}"
+                  equals:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+                    kubernetes.namespace: kube-system
-        - type: kubernetes
+                config:
-          templates:
+                  - type: docker
-            - condition:
+                    containers.ids:
-                equals:
+                      - "${data.kubernetes.container.id}"
-                  kubernetes.namespace: openstack
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-              config:
+          - type: kubernetes
-                - type: docker
+            templates:
-                  containers.ids:
+              - condition:
-                    - "${data.kubernetes.container.id}"
+                  equals:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+                    kubernetes.namespace: ceph
-    prospectors:
+                config:
-      - type: docker
+                  - type: docker
-        containers.ids:
+                    containers.ids:
-          - "*"
+                      - "${data.kubernetes.container.id}"
-        multiline:
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-          pattern: '^Traceback'
+          - type: kubernetes
-          match: after
+            templates:
-          negate: true
+              - condition:
-    processors:
+                  equals:
-      - add_kubernetes_metadata:
+                    kubernetes.namespace: openstack
-          in_cluster: true
+                config:
-      - drop_event:
+                  - type: docker
-          when:
+                    containers.ids:
-            equals:
+                      - "${data.kubernetes.container.id}"
-              kubernetes:
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-                container:
+          - type: kubernetes
-                  name: "filebeat"
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: osh-infra
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+      processors:
+        - add_kubernetes_metadata:
+            in_cluster: true
+        - drop_event:
+            when:
+              equals:
+                kubernetes:
+                  container:
+                    name: "filebeat"
+  modules:
+    system:
+      - module: system
+        syslog:
+          enabled: true
+          var.paths: ["/var/log/syslog*"]
+          fields:
+            host:
+              name: "${NODE_NAME}"
+        auth:
+          enabled: true
+          var.paths: ["/var/log/auth.log"]
+          fields:
+            host:
+              name: "${NODE_NAME}"
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -165,6 +189,23 @@ endpoints:
     port:
       http:
         default: 80
+  kibana:
+    name: kibana
+    namespace: null
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80
   filebeat:
     namespace: null
     name: filebeat

elastic-metricbeat/templates/daemonset-node-metrics.yaml

@@ -48,8 +48,10 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
+      - events
     verbs:
       - get
       - list
@@ -104,7 +106,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           args:
             - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
             - "-e"
             - "-system.hostfs=/hostfs"
           env:
@@ -112,6 +114,10 @@ spec:
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -130,7 +136,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
               subPath: metricbeat.yml
               readOnly: true
             - name: metricbeat-etc

elastic-metricbeat/templates/deployment-modules.yaml

@@ -19,9 +19,54 @@ limitations under the License.
 
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 
-{{- $serviceAccountName := "metricbeat" }}
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }}
 {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - nodes
+      - pods
+      - services
+      - endpoints
+      - replicationcontrollers
+      - limitranges
+      - events
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - get
+      - list
+      - watch
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -57,7 +102,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           args:
             - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
             - "-e"
           env:
             - name: ELASTICSEARCH_HOST
@@ -68,6 +113,10 @@ spec:
               value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: KUBE_STATE_METRICS_PORT
               value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -86,7 +135,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
               subPath: metricbeat.yml
               readOnly: true
             - name: metricbeat-etc

elastic-metricbeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    metricbeat: docker.elastic.co/beats/metricbeat:6.3.1
+    metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -53,7 +53,9 @@ dependencies:
             service: local_image_registry
   static:
     metricbeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
     image_repo_sync:
       services:
         - endpoint: internal
@@ -64,9 +66,14 @@ conf:
     setup:
       dashboards:
         enabled: true
+        index: metricbeat-*
         retry:
           enabled: true
           interval: 5
+      kibana:
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
     metricbeat:
       config:
         modules:
@@ -79,35 +86,20 @@ conf:
         username: ${ELASTICSEARCH_USERNAME}
         password: ${ELASTICSEARCH_PASSWORD}
   modules:
-    mysql:
+    docker:
-      - module: mysql
+      - module: docker
-        metricsets: ["status"]
+        metricsets:
-        hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"]
+          - "container"
-        username: root
+          - "cpu"
-        password: password
+          - "diskio"
-    rabbitmq:
+          - "healthcheck"
-      - module: rabbitmq
+          - "info"
-        metricsets: ["node", "queue", "connection"]
+          - "image"
+          - "memory"
+          - "network"
+        hosts: ["unix:///var/run/docker.sock"]
+        period: 10s
         enabled: true
-        period: 30s
-        hosts: ["rabbitmq.openstack.svc.cluster.local:15672"]
-        username: rabbitmq
-        password: password
-    # docker:
-    #   - module: docker
-    #     metricsets:
-    #       - "container"
-    #       - "cpu"
-    #       - "diskio"
-    #       - "healthcheck"
-    #       - "info"
-    #       - "image"
-    #       - "memory"
-    #       - "network"
-    #     hosts: ["unix:///var/run/docker.sock"]
-    #     period: 10s
-    #     enabled: false
-    #     labels.dedot: true
     system:
       - module: system
         period: 10s
@@ -121,18 +113,15 @@ conf:
           - core
           - diskio
           - socket
+          - filesystem
+          - fsstat
         processes: ['.*']
+        cpu.metrics:  ["percentages"]
+        core.metrics: ["percentages"]
         process.include_top_n:
           by_cpu: 5
           by_memory: 5
-      - module: system
+        enabled: true
-        period: 1m
-        metricsets:
-          - filesystem
-          - fsstat
-        processors:
-        - drop_event.when.regexp:
-            system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
     daemonset_kubernetes:
       - module: kubernetes
         metricsets:
@@ -143,6 +132,9 @@ conf:
           - volume
         period: 10s
         hosts: ["localhost:10255"]
+        add_metadata: true
+        in_cluster: true
+        enabled: true
     deployment_kubernetes:
       - module: kubernetes
         metricsets:
@@ -154,6 +146,9 @@ conf:
           - event
         period: 10s
         hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}']
+        add_metadata: true
+        in_cluster: true
+        enabled: true
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -203,6 +198,23 @@ endpoints:
     port:
       http:
         default: 80
+  kibana:
+    name: kibana
+    namespace: osh-infra
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80
 
 pod:
   affinity:

elastic-packetbeat/templates/daemonset.yaml

@@ -48,6 +48,7 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
     verbs:
@@ -102,7 +103,7 @@ spec:
                 - NET_ADMIN
           args:
             - "-c"
-            - "/etc/packetbeat/packetbeat.yml"
+            - "/usr/share/packetbeat/packetbeat.yml"
             - "-e"
           env:
             - name: ELASTICSEARCH_HOST
@@ -131,7 +132,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: packetbeat-etc
-              mountPath: /etc/packetbeat/packetbeat.yml
+              mountPath: /usr/share/packetbeat/packetbeat.yml
               subPath: packetbeat.yml
               readOnly: true
 {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }}

elastic-packetbeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    packetbeat: docker.elastic.co/beats/packetbeat:6.3.1
+    packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -63,28 +63,30 @@ conf:
   packetbeat:
     setup:
       kibana:
-        host: ['${KIBANA_HOST}:${KIBANA_PORT}']
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
       dashboards:
         enabled: true
+        index: "packetbeat-*"
         retry:
           enabled: true
           interval: 5
     packetbeat:
+      flows:
+        timeout: 30s
+        period: 10s
       interfaces:
         device: any
       protocols:
+        - type: dhcpv4
+          ports: [67, 68]
         - type: dns
           ports: [53]
           include_authorities: true
           include_additionals: true
         - type: http
-          ports: [80, 8080, 8081, 5000, 8002]
+          ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134]
-        - type: memcache
-          ports: [11211]
-        - type: mysql
-          ports: [3306]
-        - type: rabbit
-          ports: [5672, 15672]
     output:
       elasticsearch:
         hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}']

tools/deployment/elastic-beats/005-deploy-k8s.sh

@@ -0,0 +1 @@
+../common/005-deploy-k8s.sh

tools/deployment/elastic-beats/010-deploy-docker-registry.sh

@@ -1 +0,0 @@
-../multinode/010-deploy-docker-registry.sh

tools/deployment/elastic-beats/050-elasticsearch.sh

@@ -21,12 +21,39 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+manifests:
+  cron_curator: false
+  configmap_bin_curator: false
+  configmap_etc_curator: false
+images:
+  tags:
+    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115
+storage:
+  data:
+    requests:
+      storage: 20Gi
+  master:
+    requests:
+      storage: 5Gi
+jobs:
+  verify_repositories:
+    cron: "*/10 * * * *"
 monitoring:
   prometheus:
-    enabled: true
+    enabled: false
 pod:
   replicas:
+    client: 1
     data: 1
+    master: 2
+conf:
+  elasticsearch:
+    config:
+      xpack:
+        security:
+          enabled: false
+        ilm:
+          enabled: false
 
 EOF
 helm upgrade --install elasticsearch ./elasticsearch \

tools/deployment/elastic-beats/060-kibana.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make kibana
+
+: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"}
+
+tee /tmp/kibana.yaml << EOF
+images:
+  tags:
+    kibana: docker.elastic.co/kibana/kibana:7.1.0
+conf:
+  kibana:
+    xpack:
+      security:
+        enabled: false
+      spaces:
+        enabled: false
+      apm:
+        enabled: false
+      graph:
+        enabled: false
+      ml:
+        enabled: false
+      monitoring:
+        enabled: false
+      reporting:
+        enabled: false
+      canvas:
+        enabled: false
+EOF
+
+#NOTE: Deploy command
+helm upgrade --install kibana ./kibana \
+  --namespace=osh-infra \
+  --values=/tmp/kibana.yaml
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status kibana

tools/deployment/elastic-beats/080-elastic-metricbeat.sh

@@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-metricbeat
 
+tee /tmp/metricbeat.yaml << EOF
+images:
+  tags:
+    metricbeat: docker.elastic.co/beats/metricbeat:7.1.0
+conf:
+  metricbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-metricbeat ./elastic-metricbeat \
     --namespace=kube-system \
-    --set endpoints.kube_state_metrics.namespace=kube-system \
+    --values=/tmp/metricbeat.yaml
-    --set endpoints.elasticsearch.namespace=osh-infra
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/090-elastic-filebeat.sh

@@ -19,10 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-filebeat
 
+tee /tmp/filebeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/filebeat:7.1.0
+conf:
+  filebeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-filebeat ./elastic-filebeat \
     --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
+    --values=/tmp/filebeat.yaml
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/100-elastic-apm-server.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Openstack-Helm Authors.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -xe
-
-#NOTE: Lint and package chart
-make elastic-apm-server
-
-#NOTE: Deploy command
-helm upgrade --install elastic-apm-server ./elastic-apm-server \
-    --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
-
-#NOTE: Wait for deploy
-./tools/deployment/common/wait-for-pods.sh kube-system
-
-#NOTE: Validate Deployment info
-helm status elastic-apm-server

tools/deployment/elastic-beats/100-elastic-packetbeat.sh

@@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-packetbeat
 
+tee /tmp/packetbeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/packetbeat:7.1.0
+conf:
+  packetbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-packetbeat ./elastic-packetbeat \
     --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra \
+    --values=/tmp/packetbeat.yaml
-    --set endpoints.kibana.namespace=osh-infra
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/110-kibana.sh

@@ -1 +0,0 @@
-../osh-infra-logging/075-kibana.sh

zuul.d/jobs.yaml

@@ -347,29 +347,24 @@
 - job:
     name: openstack-helm-infra-elastic-beats
     parent: openstack-helm-infra-functional
-    nodeset: openstack-helm-five-node-ubuntu
     timeout: 7200
-    pre-run:
+    pre-run: playbooks/osh-infra-upgrade-host.yaml
-      - playbooks/osh-infra-upgrade-host.yaml
-      - playbooks/osh-infra-deploy-docker.yaml
-      - playbooks/osh-infra-build.yaml
-      - playbooks/osh-infra-deploy-k8s.yaml
     run: playbooks/osh-infra-gate-runner.yaml
     post-run: playbooks/osh-infra-collect-logs.yaml
+    nodeset: openstack-helm-single-node
     vars:
       gate_scripts:
-        - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh
+        - ./tools/deployment/elastic-beats/005-deploy-k8s.sh
         - ./tools/deployment/elastic-beats/020-ingress.sh
         - ./tools/deployment/elastic-beats/030-ceph.sh
         - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh
         - ./tools/deployment/elastic-beats/040-ldap.sh
         - ./tools/deployment/elastic-beats/050-elasticsearch.sh
-        - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh
+        - ./tools/deployment/elastic-beats/060-kibana.sh
         - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh
-        - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh
+        - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh
-        - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh
+        - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh
-        - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh
+        - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh
-        - ./tools/deployment/elastic-beats/110-kibana.sh
 
 - job:
     name: openstack-helm-infra-armada-deploy

zuul.d/project.yaml

@@ -59,15 +59,15 @@
         # - openstack-helm-infra-armada-deploy
         # - openstack-helm-infra-armada-update-uuid
         # - openstack-helm-infra-armada-update-passwords
-    # experimental:
+    experimental:
-    #   jobs:
+      jobs:
         # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved
         #- openstack-helm-infra-five-fedora
         # NOTE(srwilkers): Disable centos experimental jobs until issues resolved
         #- openstack-helm-infra-five-centos
         # - openstack-helm-infra-five-ubuntu
+        - openstack-helm-infra-elastic-beats
         # - openstack-helm-infra-tenant-ceph
-        # - openstack-helm-infra-elastic-beats
         # - openstack-helm-infra-armada-deploy
         # - openstack-helm-infra-armada-update-uuid
         # - openstack-helm-infra-armada-update-passwords