Update Elastic Beats versions to 7.1.0

This updates the Elastic Beats charts to 7.1.0 to keep them
aligned with the Kibana and Elasticsearch chart versions, which
is required for compatibility

This also updates the experimental job to use the single node
minikube deployment as opposed to the standard 5 node multinode
deployment

Change-Id: I4baba6ca2ea2f3785f11905138b67979a4501caa
Signed-off-by: Steve Wilkerson <sw5822@att.com>

elastic-filebeat/templates/configmap-etc.yaml

@@ -23,5 +23,7 @@ metadata:
   name: filebeat-etc
 data:
   filebeat.yml: |
-{{ toYaml .Values.conf | indent 4 }}
+{{ toYaml .Values.conf.filebeat | indent 4 }}
+  system.yml: |
+{{ toYaml .Values.conf.modules.system | indent 4 }}
 {{- end }}

elastic-filebeat/templates/daemonset.yaml

@@ -48,6 +48,7 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
     verbs:
@@ -105,10 +106,18 @@ spec:
             - name: filebeat
               containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: ELASTICSEARCH_HOST
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -133,6 +142,10 @@ spec:
               mountPath: /usr/share/filebeat/filebeat.yml
               readOnly: true
               subPath: filebeat.yml
+            - name: filebeat-etc
+              mountPath: /usr/share/filebeat/modules.d/system.yml
+              subPath: system.yml
+              readOnly: true
 {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }}
       volumes:
         - name: pod-tmp

elastic-filebeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    filebeat: docker.elastic.co/beats/filebeat:6.2.3
+    filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -53,82 +53,106 @@ dependencies:
             service: local_image_registry
   static:
     filebeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
     image_repo_sync:
       services:
         - endpoint: internal
           service: local_image_registry
 
 conf:
-  setup:
-    dashboards:
-      enabled: true
-  path:
-    logs: /var/log/
-  output:
-    elasticsearch:
-      hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
-      username: "${ELASTICSEARCH_USERNAME}"
-      password: "${ELASTICSEARCH_PASSWORD}"
   filebeat:
-    config:
+    setup:
-      prospectors:
+      dashboards:
-        path: ${path.config}/prospectors.d/*.yml
+        enabled: true
-        reload:
+        index: "filebeat-*"
-          enabled: false
+        retry:
-      modules:
+          enabled: true
-        path: ${path.config}/modules.d/*.yml
+          interval: 5
-        reload:
+      kibana:
-          enabled: false
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
-    autodiscover:
+        username: "${ELASTICSEARCH_USERNAME}"
-      providers:
+        password: "${ELASTICSEARCH_PASSWORD}"
-        - type: kubernetes
+    path:
-          templates:
+      logs: /var/log/
-            - condition:
+    output:
-                equals:
+      elasticsearch:
-                  kubernetes.namespace: kube-system
+        hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"]
-              config:
+        username: "${ELASTICSEARCH_USERNAME}"
-                - type: docker
+        password: "${ELASTICSEARCH_PASSWORD}"
-                  containers.ids:
+    filebeat:
-                    - "${data.kubernetes.container.id}"
+      config:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+        modules:
-        - type: kubernetes
+          path: ${path.config}/modules.d/*.yml
-          templates:
+          reload:
-            - condition:
+            enabled: true
-                equals:
+      autodiscover:
-                  kubernetes.namespace: ceph
+        providers:
-              config:
+          - type: kubernetes
-                - type: docker
+            templates:
-                  containers.ids:
+              - condition:
-                    - "${data.kubernetes.container.id}"
+                  equals:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+                    kubernetes.namespace: kube-system
-        - type: kubernetes
+                config:
-          templates:
+                  - type: docker
-            - condition:
+                    containers.ids:
-                equals:
+                      - "${data.kubernetes.container.id}"
-                  kubernetes.namespace: openstack
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-              config:
+          - type: kubernetes
-                - type: docker
+            templates:
-                  containers.ids:
+              - condition:
-                    - "${data.kubernetes.container.id}"
+                  equals:
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
+                    kubernetes.namespace: ceph
-    prospectors:
+                config:
-      - type: docker
+                  - type: docker
-        containers.ids:
+                    containers.ids:
-          - "*"
+                      - "${data.kubernetes.container.id}"
-        multiline:
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-          pattern: '^Traceback'
+          - type: kubernetes
-          match: after
+            templates:
-          negate: true
+              - condition:
-    processors:
+                  equals:
-      - add_kubernetes_metadata:
+                    kubernetes.namespace: openstack
-          in_cluster: true
+                config:
-      - drop_event:
+                  - type: docker
-          when:
+                    containers.ids:
-            equals:
+                      - "${data.kubernetes.container.id}"
-              kubernetes:
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
-                container:
+          - type: kubernetes
-                  name: "filebeat"
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: osh-infra
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+      processors:
+        - add_kubernetes_metadata:
+            in_cluster: true
+        - drop_event:
+            when:
+              equals:
+                kubernetes:
+                  container:
+                    name: "filebeat"
+  modules:
+    system:
+      - module: system
+        syslog:
+          enabled: true
+          var.paths: ["/var/log/syslog*"]
+          fields:
+            host:
+              name: "${NODE_NAME}"
+        auth:
+          enabled: true
+          var.paths: ["/var/log/auth.log"]
+          fields:
+            host:
+              name: "${NODE_NAME}"
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -165,6 +189,23 @@ endpoints:
     port:
       http:
         default: 80
+  kibana:
+    name: kibana
+    namespace: null
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80
   filebeat:
     namespace: null
     name: filebeat

elastic-metricbeat/templates/daemonset-node-metrics.yaml

@@ -48,8 +48,10 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
+      - events
     verbs:
       - get
       - list
@@ -104,7 +106,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           args:
             - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
             - "-e"
             - "-system.hostfs=/hostfs"
           env:
@@ -112,6 +114,10 @@ spec:
               value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: ELASTICSEARCH_PORT
               value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -130,7 +136,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
               subPath: metricbeat.yml
               readOnly: true
             - name: metricbeat-etc

elastic-metricbeat/templates/deployment-modules.yaml

@@ -19,9 +19,54 @@ limitations under the License.
 
 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}
 
-{{- $serviceAccountName := "metricbeat" }}
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }}
 {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - nodes
+      - pods
+      - services
+      - endpoints
+      - replicationcontrollers
+      - limitranges
+      - events
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - get
+      - list
+      - watch
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -57,7 +102,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           args:
             - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
             - "-e"
           env:
             - name: ELASTICSEARCH_HOST
@@ -68,6 +113,10 @@ spec:
               value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
             - name: KUBE_STATE_METRICS_PORT
               value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
             - name: ELASTICSEARCH_USERNAME
               valueFrom:
                 secretKeyRef:
@@ -86,7 +135,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
               subPath: metricbeat.yml
               readOnly: true
             - name: metricbeat-etc

elastic-metricbeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    metricbeat: docker.elastic.co/beats/metricbeat:6.3.1
+    metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -53,7 +53,9 @@ dependencies:
             service: local_image_registry
   static:
     metricbeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
     image_repo_sync:
       services:
         - endpoint: internal
@@ -64,9 +66,14 @@ conf:
     setup:
       dashboards:
         enabled: true
+        index: metricbeat-*
         retry:
           enabled: true
           interval: 5
+      kibana:
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
     metricbeat:
       config:
         modules:
@@ -79,35 +86,20 @@ conf:
         username: ${ELASTICSEARCH_USERNAME}
         password: ${ELASTICSEARCH_PASSWORD}
   modules:
-    mysql:
+    docker:
-      - module: mysql
+      - module: docker
-        metricsets: ["status"]
+        metricsets:
-        hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"]
+          - "container"
-        username: root
+          - "cpu"
-        password: password
+          - "diskio"
-    rabbitmq:
+          - "healthcheck"
-      - module: rabbitmq
+          - "info"
-        metricsets: ["node", "queue", "connection"]
+          - "image"
+          - "memory"
+          - "network"
+        hosts: ["unix:///var/run/docker.sock"]
+        period: 10s
         enabled: true
-        period: 30s
-        hosts: ["rabbitmq.openstack.svc.cluster.local:15672"]
-        username: rabbitmq
-        password: password
-    # docker:
-    #   - module: docker
-    #     metricsets:
-    #       - "container"
-    #       - "cpu"
-    #       - "diskio"
-    #       - "healthcheck"
-    #       - "info"
-    #       - "image"
-    #       - "memory"
-    #       - "network"
-    #     hosts: ["unix:///var/run/docker.sock"]
-    #     period: 10s
-    #     enabled: false
-    #     labels.dedot: true
     system:
       - module: system
         period: 10s
@@ -121,18 +113,15 @@ conf:
           - core
           - diskio
           - socket
+          - filesystem
+          - fsstat
         processes: ['.*']
+        cpu.metrics:  ["percentages"]
+        core.metrics: ["percentages"]
         process.include_top_n:
           by_cpu: 5
           by_memory: 5
-      - module: system
+        enabled: true
-        period: 1m
-        metricsets:
-          - filesystem
-          - fsstat
-        processors:
-        - drop_event.when.regexp:
-            system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
     daemonset_kubernetes:
       - module: kubernetes
         metricsets:
@@ -143,6 +132,9 @@ conf:
           - volume
         period: 10s
         hosts: ["localhost:10255"]
+        add_metadata: true
+        in_cluster: true
+        enabled: true
     deployment_kubernetes:
       - module: kubernetes
         metricsets:
@@ -154,6 +146,9 @@ conf:
           - event
         period: 10s
         hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}']
+        add_metadata: true
+        in_cluster: true
+        enabled: true
 
 endpoints:
   cluster_domain_suffix: cluster.local
@@ -203,6 +198,23 @@ endpoints:
     port:
       http:
         default: 80
+  kibana:
+    name: kibana
+    namespace: osh-infra
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80
 
 pod:
   affinity:

elastic-packetbeat/templates/daemonset.yaml

@@ -48,6 +48,7 @@ rules:
       - nodes
       - pods
       - services
+      - endpoints
       - replicationcontrollers
       - limitranges
     verbs:
@@ -102,7 +103,7 @@ spec:
                 - NET_ADMIN
           args:
             - "-c"
-            - "/etc/packetbeat/packetbeat.yml"
+            - "/usr/share/packetbeat/packetbeat.yml"
             - "-e"
           env:
             - name: ELASTICSEARCH_HOST
@@ -131,7 +132,7 @@ spec:
             - name: pod-tmp
               mountPath: /tmp
             - name: packetbeat-etc
-              mountPath: /etc/packetbeat/packetbeat.yml
+              mountPath: /usr/share/packetbeat/packetbeat.yml
               subPath: packetbeat.yml
               readOnly: true
 {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }}

elastic-packetbeat/values.yaml

@@ -28,7 +28,7 @@ labels:
 
 images:
   tags:
-    packetbeat: docker.elastic.co/beats/packetbeat:6.3.1
+    packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
     image_repo_sync: docker.io/docker:17.07.0
   pull_policy: IfNotPresent
@@ -63,28 +63,30 @@ conf:
   packetbeat:
     setup:
       kibana:
-        host: ['${KIBANA_HOST}:${KIBANA_PORT}']
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
       dashboards:
         enabled: true
+        index: "packetbeat-*"
         retry:
           enabled: true
           interval: 5
     packetbeat:
+      flows:
+        timeout: 30s
+        period: 10s
       interfaces:
         device: any
       protocols:
+        - type: dhcpv4
+          ports: [67, 68]
         - type: dns
           ports: [53]
           include_authorities: true
           include_additionals: true
         - type: http
-          ports: [80, 8080, 8081, 5000, 8002]
+          ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134]
-        - type: memcache
-          ports: [11211]
-        - type: mysql
-          ports: [3306]
-        - type: rabbit
-          ports: [5672, 15672]
     output:
       elasticsearch:
         hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}']

tools/deployment/elastic-beats/005-deploy-k8s.sh

@@ -0,0 +1 @@
+../common/005-deploy-k8s.sh

tools/deployment/elastic-beats/010-deploy-docker-registry.sh

@@ -1 +0,0 @@
-../multinode/010-deploy-docker-registry.sh

tools/deployment/elastic-beats/050-elasticsearch.sh

@@ -21,12 +21,39 @@ make elasticsearch
 
 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+manifests:
+  cron_curator: false
+  configmap_bin_curator: false
+  configmap_etc_curator: false
+images:
+  tags:
+    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115
+storage:
+  data:
+    requests:
+      storage: 20Gi
+  master:
+    requests:
+      storage: 5Gi
+jobs:
+  verify_repositories:
+    cron: "*/10 * * * *"
 monitoring:
   prometheus:
-    enabled: true
+    enabled: false
 pod:
   replicas:
+    client: 1
     data: 1
+    master: 2
+conf:
+  elasticsearch:
+    config:
+      xpack:
+        security:
+          enabled: false
+        ilm:
+          enabled: false
 
 EOF
 helm upgrade --install elasticsearch ./elasticsearch \

tools/deployment/elastic-beats/060-kibana.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make kibana
+
+: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"}
+
+tee /tmp/kibana.yaml << EOF
+images:
+  tags:
+    kibana: docker.elastic.co/kibana/kibana:7.1.0
+conf:
+  kibana:
+    xpack:
+      security:
+        enabled: false
+      spaces:
+        enabled: false
+      apm:
+        enabled: false
+      graph:
+        enabled: false
+      ml:
+        enabled: false
+      monitoring:
+        enabled: false
+      reporting:
+        enabled: false
+      canvas:
+        enabled: false
+EOF
+
+#NOTE: Deploy command
+helm upgrade --install kibana ./kibana \
+  --namespace=osh-infra \
+  --values=/tmp/kibana.yaml
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status kibana

tools/deployment/elastic-beats/080-elastic-metricbeat.sh

@@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-metricbeat
 
+tee /tmp/metricbeat.yaml << EOF
+images:
+  tags:
+    metricbeat: docker.elastic.co/beats/metricbeat:7.1.0
+conf:
+  metricbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-metricbeat ./elastic-metricbeat \
     --namespace=kube-system \
-    --set endpoints.kube_state_metrics.namespace=kube-system \
+    --values=/tmp/metricbeat.yaml
-    --set endpoints.elasticsearch.namespace=osh-infra
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/090-elastic-filebeat.sh

@@ -19,10 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-filebeat
 
+tee /tmp/filebeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/filebeat:7.1.0
+conf:
+  filebeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-filebeat ./elastic-filebeat \
     --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
+    --values=/tmp/filebeat.yaml
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/100-elastic-apm-server.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Openstack-Helm Authors.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -xe
-
-#NOTE: Lint and package chart
-make elastic-apm-server
-
-#NOTE: Deploy command
-helm upgrade --install elastic-apm-server ./elastic-apm-server \
-    --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
-
-#NOTE: Wait for deploy
-./tools/deployment/common/wait-for-pods.sh kube-system
-
-#NOTE: Validate Deployment info
-helm status elastic-apm-server

tools/deployment/elastic-beats/100-elastic-packetbeat.sh

@@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-packetbeat
 
+tee /tmp/packetbeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/packetbeat:7.1.0
+conf:
+  packetbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-packetbeat ./elastic-packetbeat \
     --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra \
+    --values=/tmp/packetbeat.yaml
-    --set endpoints.kibana.namespace=osh-infra
 
 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system

tools/deployment/elastic-beats/110-kibana.sh

@@ -1 +0,0 @@
-../osh-infra-logging/075-kibana.sh

zuul.d/jobs.yaml

@@ -348,29 +348,24 @@
 - job:
     name: openstack-helm-infra-elastic-beats
     parent: openstack-helm-infra-functional
-    nodeset: openstack-helm-five-node-ubuntu
     timeout: 7200
-    pre-run:
+    pre-run: playbooks/osh-infra-upgrade-host.yaml
-      - playbooks/osh-infra-upgrade-host.yaml
-      - playbooks/osh-infra-deploy-docker.yaml
-      - playbooks/osh-infra-build.yaml
-      - playbooks/osh-infra-deploy-k8s.yaml
     run: playbooks/osh-infra-gate-runner.yaml
     post-run: playbooks/osh-infra-collect-logs.yaml
+    nodeset: openstack-helm-single-node
     vars:
       gate_scripts:
-        - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh
+        - ./tools/deployment/elastic-beats/005-deploy-k8s.sh
         - ./tools/deployment/elastic-beats/020-ingress.sh
         - ./tools/deployment/elastic-beats/030-ceph.sh
         - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh
         - ./tools/deployment/elastic-beats/040-ldap.sh
         - ./tools/deployment/elastic-beats/050-elasticsearch.sh
-        - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh
+        - ./tools/deployment/elastic-beats/060-kibana.sh
         - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh
-        - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh
+        - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh
-        - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh
+        - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh
-        - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh
+        - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh
-        - ./tools/deployment/elastic-beats/110-kibana.sh
 
 - job:
     name: openstack-helm-infra-armada-deploy

zuul.d/project.yaml

@@ -57,15 +57,15 @@
         # - openstack-helm-infra-armada-deploy
         # - openstack-helm-infra-armada-update-uuid
         # - openstack-helm-infra-armada-update-passwords
-    # experimental:
+    experimental:
-    #   jobs:
+      jobs:
         # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved
         #- openstack-helm-infra-five-fedora
         # NOTE(srwilkers): Disable centos experimental jobs until issues resolved
         #- openstack-helm-infra-five-centos
         # - openstack-helm-infra-five-ubuntu
+        - openstack-helm-infra-elastic-beats
         # - openstack-helm-infra-tenant-ceph
-        # - openstack-helm-infra-elastic-beats
         # - openstack-helm-infra-armada-deploy
         # - openstack-helm-infra-armada-update-uuid
         # - openstack-helm-infra-armada-update-passwords