Update Elastic Beats versions to 7.1.0

This updates the Elastic Beats charts to 7.1.0 to keep them aligned with the Kibana and Elasticsearch chart versions, which is required for compatibility This also updates the experimental job to use the single node minikube deployment as opposed to the standard 5 node multinode deployment Change-Id: I4baba6ca2ea2f3785f11905138b67979a4501caa Signed-off-by: Steve Wilkerson <sw5822@att.com>
2019-12-18 13:44:23 -06:00 · 2019-12-18 13:44:23 -06:00 · 45ac5fbe0f
commit 45ac5fbe0f
parent a78ee39dcb
19 changed files with 401 additions and 179 deletions
--- a/elastic-filebeat/templates/configmap-etc.yaml
+++ b/elastic-filebeat/templates/configmap-etc.yaml
@ -23,5 +23,7 @@ metadata:
  name: filebeat-etc
 data:
  filebeat.yml: |
-{{ toYaml .Values.conf | indent 4 }}
+{{ toYaml .Values.conf.filebeat | indent 4 }}
+  system.yml: |
+{{ toYaml .Values.conf.modules.system | indent 4 }}
 {{- end }}
--- a/elastic-filebeat/templates/daemonset.yaml
+++ b/elastic-filebeat/templates/daemonset.yaml
@ -48,6 +48,7 @@ rules:
      - nodes
      - pods
      - services
+      - endpoints
      - replicationcontrollers
      - limitranges
    verbs:
@ -105,10 +106,18 @@ spec:
            - name: filebeat
              containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
            - name: ELASTICSEARCH_HOST
              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
            - name: ELASTICSEARCH_PORT
              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
            - name: ELASTICSEARCH_USERNAME
              valueFrom:
                secretKeyRef:
@ -133,6 +142,10 @@ spec:
              mountPath: /usr/share/filebeat/filebeat.yml
              readOnly: true
              subPath: filebeat.yml
+            - name: filebeat-etc
+              mountPath: /usr/share/filebeat/modules.d/system.yml
+              subPath: system.yml
+              readOnly: true
 {{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }}
      volumes:
        - name: pod-tmp
--- a/elastic-filebeat/values.yaml
+++ b/elastic-filebeat/values.yaml
@ -28,7 +28,7 @@ labels:

 images:
  tags:
-    filebeat: docker.elastic.co/beats/filebeat:6.2.3
+    filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0
    dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
    image_repo_sync: docker.io/docker:17.07.0
  pull_policy: IfNotPresent
@ -53,82 +53,106 @@ dependencies:
            service: local_image_registry
  static:
    filebeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
    image_repo_sync:
      services:
        - endpoint: internal
          service: local_image_registry

 conf:
-  setup:
-    dashboards:
-      enabled: true
-  path:
-    logs: /var/log/
-  output:
-    elasticsearch:
-      hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"]
-      username: "${ELASTICSEARCH_USERNAME}"
-      password: "${ELASTICSEARCH_PASSWORD}"
  filebeat:
-    config:
-      prospectors:
-        path: ${path.config}/prospectors.d/*.yml
-        reload:
-          enabled: false
-      modules:
-        path: ${path.config}/modules.d/*.yml
-        reload:
-          enabled: false
-    autodiscover:
-      providers:
-        - type: kubernetes
-          templates:
-            - condition:
-                equals:
-                  kubernetes.namespace: kube-system
-              config:
-                - type: docker
-                  containers.ids:
-                    - "${data.kubernetes.container.id}"
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
-        - type: kubernetes
-          templates:
-            - condition:
-                equals:
-                  kubernetes.namespace: ceph
-              config:
-                - type: docker
-                  containers.ids:
-                    - "${data.kubernetes.container.id}"
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
-        - type: kubernetes
-          templates:
-            - condition:
-                equals:
-                  kubernetes.namespace: openstack
-              config:
-                - type: docker
-                  containers.ids:
-                    - "${data.kubernetes.container.id}"
-                  exclude_lines: ["^\\s+[\\-`('.|_]"]
-    prospectors:
-      - type: docker
-        containers.ids:
-          - "*"
-        multiline:
-          pattern: '^Traceback'
-          match: after
-          negate: true
-    processors:
-      - add_kubernetes_metadata:
-          in_cluster: true
-      - drop_event:
-          when:
-            equals:
-              kubernetes:
-                container:
-                  name: "filebeat"
+    setup:
+      dashboards:
+        enabled: true
+        index: "filebeat-*"
+        retry:
+          enabled: true
+          interval: 5
+      kibana:
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
+    path:
+      logs: /var/log/
+    output:
+      elasticsearch:
+        hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"]
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
+    filebeat:
+      config:
+        modules:
+          path: ${path.config}/modules.d/*.yml
+          reload:
+            enabled: true
+      autodiscover:
+        providers:
+          - type: kubernetes
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: kube-system
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+          - type: kubernetes
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: ceph
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+          - type: kubernetes
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: openstack
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+          - type: kubernetes
+            templates:
+              - condition:
+                  equals:
+                    kubernetes.namespace: osh-infra
+                config:
+                  - type: docker
+                    containers.ids:
+                      - "${data.kubernetes.container.id}"
+                    exclude_lines: ["^\\s+[\\-`('.|_]"]
+      processors:
+        - add_kubernetes_metadata:
+            in_cluster: true
+        - drop_event:
+            when:
+              equals:
+                kubernetes:
+                  container:
+                    name: "filebeat"
+  modules:
+    system:
+      - module: system
+        syslog:
+          enabled: true
+          var.paths: ["/var/log/syslog*"]
+          fields:
+            host:
+              name: "${NODE_NAME}"
+        auth:
+          enabled: true
+          var.paths: ["/var/log/auth.log"]
+          fields:
+            host:
+              name: "${NODE_NAME}"

 endpoints:
  cluster_domain_suffix: cluster.local
@ -165,6 +189,23 @@ endpoints:
    port:
      http:
        default: 80
+  kibana:
+    name: kibana
+    namespace: null
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80
  filebeat:
    namespace: null
    name: filebeat
--- a/elastic-metricbeat/templates/daemonset-node-metrics.yaml
+++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml
@ -48,8 +48,10 @@ rules:
      - nodes
      - pods
      - services
+      - endpoints
      - replicationcontrollers
      - limitranges
+      - events
    verbs:
      - get
      - list
@ -104,7 +106,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
          args:
            - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
            - "-e"
            - "-system.hostfs=/hostfs"
          env:
@ -112,6 +114,10 @@ spec:
              value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
            - name: ELASTICSEARCH_PORT
              value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
            - name: ELASTICSEARCH_USERNAME
              valueFrom:
                secretKeyRef:
@ -130,7 +136,7 @@ spec:
            - name: pod-tmp
              mountPath: /tmp
            - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
              subPath: metricbeat.yml
              readOnly: true
            - name: metricbeat-etc
--- a/elastic-metricbeat/templates/deployment-modules.yaml
+++ b/elastic-metricbeat/templates/deployment-modules.yaml
@ -19,9 +19,54 @@ limitations under the License.

 {{- $esUserSecret := .Values.secrets.elasticsearch.user }}

-{{- $serviceAccountName := "metricbeat" }}
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }}
 {{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - nodes
+      - pods
+      - services
+      - endpoints
+      - replicationcontrollers
+      - limitranges
+      - events
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - get
+      - list
+      - watch
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@ -57,7 +102,7 @@ spec:
 {{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
          args:
            - "-c"
-            - "/etc/metricbeat.yml"
+            - "/usr/share/metricbeat/metricbeat.yml"
            - "-e"
          env:
            - name: ELASTICSEARCH_HOST
@ -68,6 +113,10 @@ spec:
              value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
            - name: KUBE_STATE_METRICS_PORT
              value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: KIBANA_HOST
+              value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }}
+            - name: KIBANA_PORT
+              value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
            - name: ELASTICSEARCH_USERNAME
              valueFrom:
                secretKeyRef:
@ -86,7 +135,7 @@ spec:
            - name: pod-tmp
              mountPath: /tmp
            - name: metricbeat-etc
-              mountPath: /etc/metricbeat.yml
+              mountPath: /usr/share/metricbeat/metricbeat.yml
              subPath: metricbeat.yml
              readOnly: true
            - name: metricbeat-etc
--- a/elastic-metricbeat/values.yaml
+++ b/elastic-metricbeat/values.yaml
@ -28,7 +28,7 @@ labels:

 images:
  tags:
-    metricbeat: docker.elastic.co/beats/metricbeat:6.3.1
+    metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0
    dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
    image_repo_sync: docker.io/docker:17.07.0
  pull_policy: IfNotPresent
@ -53,7 +53,9 @@ dependencies:
            service: local_image_registry
  static:
    metricbeat:
-      services: null
+      services:
+        - endpoint: internal
+          service: kibana
    image_repo_sync:
      services:
        - endpoint: internal
@ -64,9 +66,14 @@ conf:
    setup:
      dashboards:
        enabled: true
+        index: metricbeat-*
        retry:
          enabled: true
          interval: 5
+      kibana:
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
    metricbeat:
      config:
        modules:
@ -79,35 +86,20 @@ conf:
        username: ${ELASTICSEARCH_USERNAME}
        password: ${ELASTICSEARCH_PASSWORD}
  modules:
-    mysql:
-      - module: mysql
-        metricsets: ["status"]
-        hosts: ["tcp(mariadb.openstack.svc.cluster.local:3306)/"]
-        username: root
-        password: password
-    rabbitmq:
-      - module: rabbitmq
-        metricsets: ["node", "queue", "connection"]
+    docker:
+      - module: docker
+        metricsets:
+          - "container"
+          - "cpu"
+          - "diskio"
+          - "healthcheck"
+          - "info"
+          - "image"
+          - "memory"
+          - "network"
+        hosts: ["unix:///var/run/docker.sock"]
+        period: 10s
        enabled: true
-        period: 30s
-        hosts: ["rabbitmq.openstack.svc.cluster.local:15672"]
-        username: rabbitmq
-        password: password
-    # docker:
-    #   - module: docker
-    #     metricsets:
-    #       - "container"
-    #       - "cpu"
-    #       - "diskio"
-    #       - "healthcheck"
-    #       - "info"
-    #       - "image"
-    #       - "memory"
-    #       - "network"
-    #     hosts: ["unix:///var/run/docker.sock"]
-    #     period: 10s
-    #     enabled: false
-    #     labels.dedot: true
    system:
      - module: system
        period: 10s
@ -121,18 +113,15 @@ conf:
          - core
          - diskio
          - socket
+          - filesystem
+          - fsstat
        processes: ['.*']
+        cpu.metrics:  ["percentages"]
+        core.metrics: ["percentages"]
        process.include_top_n:
          by_cpu: 5
          by_memory: 5
-      - module: system
-        period: 1m
-        metricsets:
-          - filesystem
-          - fsstat
-        processors:
-        - drop_event.when.regexp:
-            system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
+        enabled: true
    daemonset_kubernetes:
      - module: kubernetes
        metricsets:
@ -143,6 +132,9 @@ conf:
          - volume
        period: 10s
        hosts: ["localhost:10255"]
+        add_metadata: true
+        in_cluster: true
+        enabled: true
    deployment_kubernetes:
      - module: kubernetes
        metricsets:
@ -154,6 +146,9 @@ conf:
          - event
        period: 10s
        hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}']
+        add_metadata: true
+        in_cluster: true
+        enabled: true

 endpoints:
  cluster_domain_suffix: cluster.local
@ -203,6 +198,23 @@ endpoints:
    port:
      http:
        default: 80
+  kibana:
+    name: kibana
+    namespace: osh-infra
+    hosts:
+      default: kibana-dash
+      public: kibana
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      kibana:
+        default: 5601
+      http:
+        default: 80

 pod:
  affinity:
--- a/elastic-packetbeat/templates/daemonset.yaml
+++ b/elastic-packetbeat/templates/daemonset.yaml
@ -48,6 +48,7 @@ rules:
      - nodes
      - pods
      - services
+      - endpoints
      - replicationcontrollers
      - limitranges
    verbs:
@ -102,7 +103,7 @@ spec:
                - NET_ADMIN
          args:
            - "-c"
-            - "/etc/packetbeat/packetbeat.yml"
+            - "/usr/share/packetbeat/packetbeat.yml"
            - "-e"
          env:
            - name: ELASTICSEARCH_HOST
@ -131,7 +132,7 @@ spec:
            - name: pod-tmp
              mountPath: /tmp
            - name: packetbeat-etc
-              mountPath: /etc/packetbeat/packetbeat.yml
+              mountPath: /usr/share/packetbeat/packetbeat.yml
              subPath: packetbeat.yml
              readOnly: true
 {{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }}
--- a/elastic-packetbeat/values.yaml
+++ b/elastic-packetbeat/values.yaml
@ -28,7 +28,7 @@ labels:

 images:
  tags:
-    packetbeat: docker.elastic.co/beats/packetbeat:6.3.1
+    packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0
    dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
    image_repo_sync: docker.io/docker:17.07.0
  pull_policy: IfNotPresent
@ -63,28 +63,30 @@ conf:
  packetbeat:
    setup:
      kibana:
-        host: ['${KIBANA_HOST}:${KIBANA_PORT}']
+        host: "${KIBANA_HOST}:${KIBANA_PORT}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
      dashboards:
        enabled: true
+        index: "packetbeat-*"
        retry:
          enabled: true
          interval: 5
    packetbeat:
+      flows:
+        timeout: 30s
+        period: 10s
      interfaces:
        device: any
      protocols:
+        - type: dhcpv4
+          ports: [67, 68]
        - type: dns
          ports: [53]
          include_authorities: true
          include_additionals: true
        - type: http
-          ports: [80, 8080, 8081, 5000, 8002]
-        - type: memcache
-          ports: [11211]
-        - type: mysql
-          ports: [3306]
-        - type: rabbit
-          ports: [5672, 15672]
+          ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134]
    output:
      elasticsearch:
        hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}']
--- a/tools/deployment/elastic-beats/005-deploy-k8s.sh
+++ b/tools/deployment/elastic-beats/005-deploy-k8s.sh
@ -0,0 +1 @@
+../common/005-deploy-k8s.sh
--- a/tools/deployment/elastic-beats/010-deploy-docker-registry.sh
+++ b/tools/deployment/elastic-beats/010-deploy-docker-registry.sh
@ -1 +0,0 @@
-../multinode/010-deploy-docker-registry.sh
--- a/tools/deployment/elastic-beats/050-elasticsearch.sh
+++ b/tools/deployment/elastic-beats/050-elasticsearch.sh
@ -21,12 +21,39 @@ make elasticsearch

 #NOTE: Deploy command
 tee /tmp/elasticsearch.yaml << EOF
+manifests:
+  cron_curator: false
+  configmap_bin_curator: false
+  configmap_etc_curator: false
+images:
+  tags:
+    elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115
+storage:
+  data:
+    requests:
+      storage: 20Gi
+  master:
+    requests:
+      storage: 5Gi
+jobs:
+  verify_repositories:
+    cron: "*/10 * * * *"
 monitoring:
  prometheus:
-    enabled: true
+    enabled: false
 pod:
  replicas:
+    client: 1
    data: 1
+    master: 2
+conf:
+  elasticsearch:
+    config:
+      xpack:
+        security:
+          enabled: false
+        ilm:
+          enabled: false

 EOF
 helm upgrade --install elasticsearch ./elasticsearch \
--- a/tools/deployment/elastic-beats/060-kibana.sh
+++ b/tools/deployment/elastic-beats/060-kibana.sh
@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make kibana
+
+: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"}
+
+tee /tmp/kibana.yaml << EOF
+images:
+  tags:
+    kibana: docker.elastic.co/kibana/kibana:7.1.0
+conf:
+  kibana:
+    xpack:
+      security:
+        enabled: false
+      spaces:
+        enabled: false
+      apm:
+        enabled: false
+      graph:
+        enabled: false
+      ml:
+        enabled: false
+      monitoring:
+        enabled: false
+      reporting:
+        enabled: false
+      canvas:
+        enabled: false
+EOF
+
+#NOTE: Deploy command
+helm upgrade --install kibana ./kibana \
+  --namespace=osh-infra \
+  --values=/tmp/kibana.yaml
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate Deployment info
+helm status kibana
--- a/tools/deployment/elastic-beats/080-elastic-metricbeat.sh
+++ b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh
@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-metricbeat

+tee /tmp/metricbeat.yaml << EOF
+images:
+  tags:
+    metricbeat: docker.elastic.co/beats/metricbeat:7.1.0
+conf:
+  metricbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-metricbeat ./elastic-metricbeat \
    --namespace=kube-system \
-    --set endpoints.kube_state_metrics.namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
+    --values=/tmp/metricbeat.yaml

 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system
--- a/tools/deployment/elastic-beats/090-elastic-filebeat.sh
+++ b/tools/deployment/elastic-beats/090-elastic-filebeat.sh
@ -19,10 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-filebeat

+tee /tmp/filebeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/filebeat:7.1.0
+conf:
+  filebeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-filebeat ./elastic-filebeat \
    --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
+    --values=/tmp/filebeat.yaml

 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system
--- a/tools/deployment/elastic-beats/100-elastic-apm-server.sh
+++ b/tools/deployment/elastic-beats/100-elastic-apm-server.sh
@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Openstack-Helm Authors.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -xe
-
-#NOTE: Lint and package chart
-make elastic-apm-server
-
-#NOTE: Deploy command
-helm upgrade --install elastic-apm-server ./elastic-apm-server \
-    --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra
-
-#NOTE: Wait for deploy
-./tools/deployment/common/wait-for-pods.sh kube-system
-
-#NOTE: Validate Deployment info
-helm status elastic-apm-server
--- a/tools/deployment/elastic-beats/100-elastic-packetbeat.sh
+++ b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh
@ -19,11 +19,26 @@ set -xe
 #NOTE: Lint and package chart
 make elastic-packetbeat

+tee /tmp/packetbeat.yaml << EOF
+images:
+  tags:
+    filebeat: docker.elastic.co/beats/packetbeat:7.1.0
+conf:
+  packetbeat:
+    setup:
+      ilm:
+        enabled: false
+endpoints:
+  elasticsearch:
+    namespace: osh-infra
+  kibana:
+    namespace: osh-infra
+EOF
+
 #NOTE: Deploy command
 helm upgrade --install elastic-packetbeat ./elastic-packetbeat \
    --namespace=kube-system \
-    --set endpoints.elasticsearch.namespace=osh-infra \
-    --set endpoints.kibana.namespace=osh-infra
+    --values=/tmp/packetbeat.yaml

 #NOTE: Wait for deploy
 ./tools/deployment/common/wait-for-pods.sh kube-system
--- a/tools/deployment/elastic-beats/110-kibana.sh
+++ b/tools/deployment/elastic-beats/110-kibana.sh
@ -1 +0,0 @@
-../osh-infra-logging/075-kibana.sh
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -348,29 +348,24 @@
 - job:
    name: openstack-helm-infra-elastic-beats
    parent: openstack-helm-infra-functional
-    nodeset: openstack-helm-five-node-ubuntu
    timeout: 7200
-    pre-run:
-      - playbooks/osh-infra-upgrade-host.yaml
-      - playbooks/osh-infra-deploy-docker.yaml
-      - playbooks/osh-infra-build.yaml
-      - playbooks/osh-infra-deploy-k8s.yaml
+    pre-run: playbooks/osh-infra-upgrade-host.yaml
    run: playbooks/osh-infra-gate-runner.yaml
    post-run: playbooks/osh-infra-collect-logs.yaml
+    nodeset: openstack-helm-single-node
    vars:
      gate_scripts:
-        - ./tools/deployment/elastic-beats/010-deploy-docker-registry.sh
+        - ./tools/deployment/elastic-beats/005-deploy-k8s.sh
        - ./tools/deployment/elastic-beats/020-ingress.sh
        - ./tools/deployment/elastic-beats/030-ceph.sh
        - ./tools/deployment/elastic-beats/035-ceph-ns-activate.sh
        - ./tools/deployment/elastic-beats/040-ldap.sh
        - ./tools/deployment/elastic-beats/050-elasticsearch.sh
-        - ./tools/deployment/elastic-beats/060-elastic-metricbeat.sh
+        - ./tools/deployment/elastic-beats/060-kibana.sh
        - ./tools/deployment/elastic-beats/070-kube-state-metrics.sh
-        - ./tools/deployment/elastic-beats/080-elastic-filebeat.sh
-        - ./tools/deployment/elastic-beats/090-elastic-packetbeat.sh
-        - ./tools/deployment/elastic-beats/100-elastic-apm-server.sh
-        - ./tools/deployment/elastic-beats/110-kibana.sh
+        - ./tools/deployment/elastic-beats/080-elastic-metricbeat.sh
+        - ./tools/deployment/elastic-beats/090-elastic-filebeat.sh
+        - ./tools/deployment/elastic-beats/100-elastic-packetbeat.sh

 - job:
    name: openstack-helm-infra-armada-deploy
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@ -57,15 +57,15 @@
        # - openstack-helm-infra-armada-deploy
        # - openstack-helm-infra-armada-update-uuid
        # - openstack-helm-infra-armada-update-passwords
-    # experimental:
-    #   jobs:
+    experimental:
+      jobs:
        # NOTE(srwilkers): Disable fedora experimental jobs until issues resolved
        #- openstack-helm-infra-five-fedora
        # NOTE(srwilkers): Disable centos experimental jobs until issues resolved
        #- openstack-helm-infra-five-centos
        # - openstack-helm-infra-five-ubuntu
+        - openstack-helm-infra-elastic-beats
        # - openstack-helm-infra-tenant-ceph
-        # - openstack-helm-infra-elastic-beats
        # - openstack-helm-infra-armada-deploy
        # - openstack-helm-infra-armada-update-uuid
        # - openstack-helm-infra-armada-update-passwords