Create Chart to Deploy Apache Kafka

This proposes adding a kafka chart to osh-infra that aligns
with the design patterns laid out by the other charts in osh-infra
and osh.

danielqsj's kafka-exporter image is leveraged to deploy a prometheus
exporter for kafka alongside the main application if enabled in
values.yaml

Change-Id: I5997b0994fc3aef9bd1b222c373cc3a013112566
Co-Authored-By: Meghan Heisler <mh783g@att.com>

fluentd/templates/deployment-fluentd.yaml

@@ -19,8 +19,8 @@ limitations under the License.
 
 {{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }}
 
-{{- $kafkaBroker := tuple "kafka" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $kafkaBroker := tuple "kafka" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
-{{- $kafkaBrokerPort := tuple "kafka" "public" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
 {{- $kafkaBrokerURI := printf "%s:%s" $kafkaBroker $kafkaBrokerPort }}
 
 {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd"  }}

fluentd/values.yaml

@@ -366,6 +366,7 @@ endpoints:
     port:
       broker:
         default: 9092
+        public: 80
   prometheus_fluentd_exporter:
     namespace: null
     hosts:

kafka/Chart.yaml

@@ -0,0 +1,25 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Kafka
+name: kafka
+version: 0.1.0
+home: https://kafka.apache.org/
+sources:
+  - https://github.com/apache/kafka
+  - https://github.com/danielqsj/kafka_exporter
+  - https://opendev.org/openstack/openstack-helm-infra
+maintainers:
+  - name: OpenStack-Helm Authors

kafka/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0

kafka/templates/bin/_helm-test.sh.tpl

@@ -0,0 +1,118 @@
+#!/bin/bash
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+function create_topic () {
+    ./opt/kafka/bin/kafka-topics.sh \
+        --create --topic $1 \
+        --partitions $2 \
+        --replication-factor $3 \
+        --bootstrap-server $KAFKA_BROKERS
+}
+
+function describe_topic () {
+    ./opt/kafka/bin/kafka-topics.sh \
+        --describe --topic $1 \
+        --bootstrap-server $KAFKA_BROKERS
+}
+
+function produce_message () {
+    echo $2 | \
+    ./opt/kafka/bin/kafka-console-producer.sh \
+        --topic $1 \
+        --broker-list $KAFKA_BROKERS
+}
+
+function consume_messages () {
+    ./opt/kafka/bin/kafka-console-consumer.sh \
+        --topic $1 \
+        --timeout-ms 500 \
+        --from-beginning \
+        --bootstrap-server $KAFKA_BROKERS
+}
+
+function delete_partition_messages () {
+    ./opt/kafka/bin/kafka-delete-records.sh \
+        --offset-json-file $1 \
+        --bootstrap-server $KAFKA_BROKERS
+}
+
+function delete_topic () {
+    ./opt/kafka/bin/kafka-topics.sh \
+        --delete --topic $1 \
+        --bootstrap-server $KAFKA_BROKERS
+}
+
+set -e
+
+TOPIC="kafka-test"
+PARTITION_COUNT=3
+PARTITION_REPLICAS=2
+
+echo "Creating topic $TOPIC"
+create_topic $TOPIC $PARTITION_COUNT $PARTITION_REPLICAS
+describe_topic $TOPIC
+
+echo "Producing 5 messages"
+for i in {1..5}; do
+    MESSAGE="Message #$i"
+    produce_message $TOPIC "$MESSAGE"
+done
+
+echo -e "\nConsuming messages (A \"TimeoutException\" is expected, else this would consume forever)"
+consume_messages $TOPIC
+
+echo "Producing 5 more messages"
+for i in {6..10}; do
+    MESSAGE="Message #$i"
+    produce_message $TOPIC "$MESSAGE"
+done
+
+echo -e "\nCreating partition offset reset json file"
+tee /tmp/partition_offsets.json << EOF
+{
+"partitions": [
+    {
+        "topic": "$TOPIC",
+        "partition": 0,
+        "offset": -1
+    }, {
+        "topic": "$TOPIC",
+        "partition": 1,
+        "offset": -1
+    }, {
+        "topic": "$TOPIC",
+        "partition": 2,
+        "offset": -1
+    }
+],
+"version": 1
+}
+EOF
+
+echo "Resetting $TOPIC partitions (deleting messages)"
+delete_partition_messages /tmp/partition_offsets.json
+
+echo "Deleting topic $TOPIC"
+delete_topic $TOPIC
+
+if [ $(describe_topic $TOPIC | wc -l) -eq 0 ]; then
+    echo "Topic $TOPIC was deleted successfully."
+    exit 0
+else
+    echo "Topic $TOPIC was not successfully deleted."
+    exit 1
+fi

kafka/templates/bin/_kafka-probe.sh.tpl

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+
+echo ruok | nc 127.0.0.1 ${KAFKA_PORT}

kafka/templates/bin/_kafka.sh.tpl

@@ -0,0 +1,36 @@
+#!/bin/bash
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{ if not (empty .Values.conf.kafka.server_settings) }}
+{{ range $key, $value := .Values.conf.kafka.server_settings }}
+{{ $varName := printf "%s%s" "KAFKA_" ($key | upper) }}
+{{ $varValue := ternary ($value | quote) ($value | int) (kindIs "string" $value) }}
+export {{ $varName }}={{ $varValue }}
+{{ end }}
+{{ end }}
+
+COMMAND="${@:-start}"
+
+function start() {
+  ./usr/bin/start-kafka.sh
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

kafka/templates/configmap-bin.yaml

@@ -0,0 +1,33 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.configmap_bin }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-bin
+data:
+  kafka.sh: |
+{{ tuple "bin/_kafka.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kafka-liveness.sh: |
+{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  kafka-readiness.sh: |
+{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  helm-test.sh: |
+{{ tuple "bin/_helm-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end -}}

kafka/templates/helm_test_pod.yaml

@@ -0,0 +1,64 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.helm_test }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := print .Release.Name "-test" }}
+{{ tuple $envAll "test" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{.Release.Name}}-test"
+  labels:
+{{ tuple $envAll "kafka" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+    "helm.sh/hook": test-success
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+spec:
+{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
+  serviceAccountName: {{ $serviceAccountName }}
+  nodeSelector:
+    {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
+  restartPolicy: Never
+  initContainers:
+{{ tuple $envAll "test" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
+  containers:
+    - name: {{.Release.Name}}-helm-test
+{{ tuple $envAll "helm_test" | include "helm-toolkit.snippets.image" | indent 6 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.test | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
+{{ dict "envAll" $envAll "application" "test" "container" "helm_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
+      command:
+        - "/tmp/helm-test.sh"
+      env:
+        - name: KAFKA_BROKERS
+          value: "{{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
+      volumeMounts:
+        - name: pod-tmp
+          mountPath: /tmp
+        - name: kafka-bin
+          mountPath: /tmp/helm-test.sh
+          subPath: helm-test.sh
+          readOnly: true
+  volumes:
+    - name: pod-tmp
+      emptyDir: {}
+    - name: kafka-bin
+      configMap:
+        name: kafka-bin
+        defaultMode: 0555
+{{- end }}

kafka/templates/ingress-kafka.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.ingress .Values.network.kafka.ingress.public }}
+{{- $ingressOpts := dict "envAll" . "backendService" "kafka" "backendServiceType" "kafka" "backendPort" "broker" -}}
+{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
+{{- end }}

kafka/templates/job-image-repo-sync.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
+{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kafka" -}}
+{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
+{{- end }}

kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+COMMAND="${@:-start}"
+
+function start () {
+  exec /bin/kafka_exporter \
+        --kafka.server={{ tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+}
+
+function stop () {
+  kill -TERM 1
+}
+
+$COMMAND

kafka/templates/monitoring/prometheus/configmap-bin.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-exporter-bin
+data:
+  kafka-exporter.sh: |
+{{ tuple "bin/_kafka-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

kafka/templates/monitoring/prometheus/deployment.yaml

@@ -0,0 +1,88 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.deployment .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+
+{{- $kafkaExporterUserSecret := .Values.secrets.kafka_exporter.user }}
+
+{{- $serviceAccountName := "prometheus-kafka-exporter" }}
+{{ tuple $envAll "kafka_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-kafka-exporter
+  labels:
+{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  replicas: {{ .Values.pod.replicas.kafka_exporter }}
+  selector:
+    matchLabels:
+{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      nodeSelector:
+        {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka_exporter.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll "kafka_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container"  | indent 8 }}
+      containers:
+        - name: kafka-exporter
+{{ tuple $envAll "kafka_exporter" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.kafka_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "kafka_exporter" "container" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /tmp/kafka-exporter.sh
+            - start
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - /tmp/kafka-exporter.sh
+                  - stop
+          # env: {}
+          ports:
+            - name: exporter
+              containerPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            initialDelaySeconds: 20
+            periodSeconds: 10
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - name: kafka-exporter-bin
+              mountPath: /tmp/kafka-exporter.sh
+              subPath: kafka-exporter.sh
+              readOnly: true
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: kafka-exporter-bin
+          configMap:
+            name: kafka-exporter-bin
+            defaultMode: 0555
+{{- end }}

kafka/templates/monitoring/prometheus/network-policy.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.network_policy .Values.monitoring.prometheus.enabled -}}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-kafka-exporter" -}}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- end -}}

kafka/templates/monitoring/prometheus/service.yaml

@@ -0,0 +1,38 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.monitoring.prometheus.service .Values.monitoring.prometheus.enabled }}
+{{- $envAll := . }}
+{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kafka_exporter }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "kafka_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "kafka-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  annotations:
+{{- if .Values.monitoring.prometheus.enabled }}
+{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
+{{- end }}
+spec:
+  ports:
+  - name: exporter
+    port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    targetPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  selector:
+{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

kafka/templates/network_policy.yaml

@@ -0,0 +1,19 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */}}
+
+{{- if .Values.manifests.network_policy -}}
+{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kafka" -}}
+{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
+{{- end -}}

kafka/templates/secret-ingress-tls.yaml

@@ -0,0 +1,19 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_ingress_tls -}}
+{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "kafka" "backendService" "kafka" ) }}
+{{- end }}

kafka/templates/secret-kafka.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_kafka }}
+{{- $envAll := . }}
+{{- $secretName := .Values.secrets.kafka.admin }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  KAFKA_ADMIN_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }}
+  KAFKA_ADMIN_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }}
+{{- end }}

kafka/templates/service-discovery.yaml

@@ -0,0 +1,34 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service_discovery }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "kafka" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  ports:
+  - name: broker
+    targetPort: broker
+    port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  clusterIP: None
+  selector:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{- end }}

kafka/templates/service-ingress-kafka.yaml

@@ -0,0 +1,20 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.service_ingress .Values.network.kafka.ingress.public }}
+{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kafka" -}}
+{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
+{{- end }}

kafka/templates/service.yaml

@@ -0,0 +1,38 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "kafka" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  labels:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  ports:
+  - name: broker
+    port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    {{ if .Values.network.kafka.node_port.enabled }}
+    nodePort: {{ .Values.network.kafka.node_port.port }}
+    {{ end }}
+  selector:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+  {{ if .Values.network.kafka.node_port.enabled }}
+  type: NodePort
+  {{ end }}
+{{- end }}

kafka/templates/statefulset.yaml

@@ -0,0 +1,178 @@
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.statefulset }}
+{{- $envAll := . }}
+
+{{- $mounts_kafka := .Values.pod.mounts.kafka.kafka }}
+{{- $mounts_kafka_init := .Values.pod.mounts.kafka.init_container }}
+{{- $kafkaUserSecret := .Values.secrets.kafka.admin }}
+{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+
+{{- $serviceAccountName := printf "%s-%s" .Release.Name "kafka" }}
+{{ tuple $envAll "kafka" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+  - nonResourceURLs:
+      - "/metrics"
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: kafka
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  serviceName: {{ tuple "kafka" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  replicas: {{ .Values.pod.replicas.kafka }}
+  updateStrategy:
+    type: OnDelete
+  podManagementPolicy: Parallel
+  selector:
+    matchLabels:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+      annotations:
+{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
+        configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
+    spec:
+{{ dict "envAll" $envAll "application" "kafka" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      serviceAccountName: {{ $serviceAccountName }}
+      affinity:
+{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
+      nodeSelector:
+        {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }}
+      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka.timeout | default "30" }}
+      initContainers:
+{{ tuple $envAll "kafka" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+      containers:
+        - name: kafka
+          command:
+            - "/tmp/kafka.sh"
+{{ tuple $envAll "kafka" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.kafka | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "kafka" "container" "kafka" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          ports:
+            - name: broker
+              containerPort: {{ $kafkaBrokerPort }}
+          env:
+            - name: KAFKA_PORT
+              value: "{{ $kafkaBrokerPort }}"
+            - name: ZOOKEEPER_PORT
+              value: "{{ tuple "zookeeper" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
+            - name: KAFKA_ZOOKEEPER_CONNECT
+              value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
+            - name: KAFKA_LISTENERS
+              value: "PLAINTEXT://:{{$kafkaBrokerPort}}"
+            - name: KAFKA_CREATE_TOPICS
+              value: "{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.kafka.topics }}"
+          readinessProbe:
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 2
+            successThreshold: 1
+            exec:
+              command:
+              - /tmp/kafka-readiness.sh
+          livenessProbe:
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+            failureThreshold: 2
+            successThreshold: 1
+            exec:
+              command:
+              - /tmp/kafka-liveness.sh
+          volumeMounts:
+            - name: kafka-bin
+              mountPath: /tmp/kafka.sh
+              subPath: kafka.sh
+              readOnly: true
+            - name: kafka-bin
+              mountPath: /tmp/kafka-liveness.sh
+              subPath: kafka-liveness.sh
+              readOnly: true
+            - name: kafka-bin
+              mountPath: /tmp/kafka-readiness.sh
+              subPath: kafka-readiness.sh
+              readOnly: true
+            - name: data
+              mountPath: {{ .Values.conf.kafka.config.data_directory }}
+{{ if $mounts_kafka.volumeMounts }}{{ toYaml $mounts_kafka.volumeMounts | indent 12 }}{{ end }}
+      volumes:
+        - name: kafka-bin
+          configMap:
+            name: kafka-bin
+            defaultMode: 0555
+{{ if $mounts_kafka.volumes }}{{ toYaml $mounts_kafka.volumes | indent 8 }}{{ end }}
+{{- if not .Values.storage.enabled }}
+        - name: data
+          emptyDir: {}
+{{- else }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: {{ .Values.storage.pvc.access_mode }}
+        resources:
+          requests:
+            storage: {{ .Values.storage.requests.storage  }}
+        storageClassName: {{ .Values.storage.storage_class }}
+{{- end }}
+{{- end }}

kafka/values.yaml

@@ -0,0 +1,300 @@
+# Copyright 2019 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for kafka.
+# This is a YAML-formatted file.
+# Declare name/value pairs to be passed into your templates.
+# name: value
+
+images:
+  tags:
+    kafka: docker.io/wurstmeister/kafka:2.12-2.3.0
+    kafka_exporter: docker.io/danielqsj/kafka-exporter:latest
+    dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
+    image_repo_sync: docker.io/docker:17.07.0
+    helm_test: docker.io/wurstmeister/kafka:2.12-2.3.0
+  pull_policy: IfNotPresent
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+labels:
+  kafka:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+  test:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+
+pod:
+  security_context:
+    kafka:
+      pod: {}
+      container:
+        kafka: {}
+        kafka-init: {}
+    kafka_exporter:
+      pod: {}
+      container:
+        kafka_exporter: {}
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+      weight:
+        default: 10
+  mounts:
+    kafka:
+      kafka:
+      init_container: null
+  replicas:
+    kafka: 3
+    kafka_exporter: 1
+  lifecycle:
+    upgrades:
+      statefulsets:
+        pod_replacement_strategy: RollingUpdate
+    termination_grace_period:
+      kafka:
+        timeout: 30
+      kafka_exporter:
+        timeout: 30
+  resources:
+    enabled: false
+    kafka:
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+      requests:
+        memory: "128Mi"
+        cpu: "500m"
+    kafka_exporter:
+      requests:
+        memory: "128Mi"
+        cpu: "100m"
+      limits:
+        memory: "1024Mi"
+        cpu: "2000m"
+    jobs:
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      test:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  kafka:
+    name: kafka
+    namespace: null
+    auth:
+      admin:
+        username: admin
+        password: changeme
+    hosts:
+      default: kafka-broker
+      discovery: kafka-discovery
+      public: kafka
+    host_fqdn_override:
+      default: null
+      # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public
+      # endpoints using the following format:
+      # public:
+      #   host: null
+      #   tls:
+      #     crt: null
+      #     key: null
+    path:
+      default: null
+    scheme:
+      default: 'http'
+    port:
+      broker:
+        default: 9092
+      kafka-exporter:
+        default: 9141
+      jmx-exporter:
+        default: 9404
+  kafka_exporter:
+    namespace: null
+    hosts:
+      default: kafka-exporter
+    host_fqdn_override:
+      default: null
+    scheme:
+      default: 'http'
+    port:
+      exporter:
+        default: 9308
+  zookeeper:
+    name: zookeeper
+    namespace: null
+    auth:
+      admin:
+        username: admin
+        password: changeme
+    hosts:
+      default: zookeeper-int
+      public: zookeeper
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: 'http'
+    port:
+      client:
+        default: 2181
+      server:
+        default: 2888
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - kafka-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+    kafka:
+      services:
+        - endpoint: internal
+          service: zookeeper-int
+    kafka_exporter:
+      services:
+        - endpoint: internal
+          service: kafka-broker
+
+monitoring:
+  prometheus:
+    enabled: true
+    kafka_exporter:
+      scrape: true
+
+network:
+  kafka:
+    ingress:
+      public: true
+      classes:
+        namespace: "nginx"
+        cluster: "nginx-cluster"
+      annotations:
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/affinity: cookie
+        nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-kafka
+        nginx.ingress.kubernetes.io/session-cookie-hash: sha1
+        nginx.ingress.kubernetes.io/session-cookie-expires: "600"
+        nginx.ingress.kubernetes.io/session-cookie-max-age: "600"
+    node_port:
+      enabled: false
+      port: 31033
+
+network_policy:
+  kafka:
+    ingress:
+      - {}
+    egress:
+      - {}
+  kafka_exporter:
+    ingress:
+      - {}
+    egress:
+      - {}
+
+secrets:
+  tls:
+    kafka:
+      kafka:
+        public: kafka-tls-public
+  kafka:
+    admin: kafka-admin-creds
+  kafka_exporter:
+    user: kafka-exporter-creds
+
+storage:
+  enabled: true
+  pvc:
+    name: kafka-pvc
+    access_mode: [ "ReadWriteOnce" ]
+  requests:
+    storage: 5Gi
+  storage_class: general
+
+manifests:
+  configmap_bin: true
+  configmap_etc: true
+  helm_test: true
+  ingress: true
+  job_image_repo_sync: true
+  monitoring:
+    prometheus:
+      configmap_bin: true
+      deployment: true
+      service: true
+      network_policy: false
+  network_policy: false
+  secret_ingress_tls: true
+  secret_kafka: true
+  secret_zookeeper: true
+  service_discovery: true
+  service_ingress: true
+  service: true
+  statefulset: true
+
+conf:
+  kafka:
+    config:
+      data_directory: /var/lib/kafka/data
+    server_settings: {}
+      # Optionally provide configuration overrides for
+      # Kafka's server.properties file ie:
+      # message_max_bytes: 5000000
+    topics: []
+      # List of topic strings formatted like:
+      # topic_name:number_of_partitions:replication_factor
+      # - "mytopic:1:1"

tools/deployment/osh-infra-kafka/000-install-packages.sh

@@ -0,0 +1 @@
+../common/000-install-packages.sh

tools/deployment/osh-infra-kafka/005-deploy-k8s.sh

@@ -0,0 +1 @@
+../common/005-deploy-k8s.sh

tools/deployment/osh-infra-kafka/010-ingress.sh

@@ -0,0 +1 @@
+../osh-infra-logging/010-ingress.sh

tools/deployment/osh-infra-kafka/020-ceph.sh

@@ -0,0 +1 @@
+../osh-infra-logging/020-ceph.sh

tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh

@@ -0,0 +1 @@
+../osh-infra-logging/025-ceph-ns-activate.sh

tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh

@@ -0,0 +1 @@
+../osh-infra-logging/030-radosgw-osh-infra.sh

tools/deployment/osh-infra-kafka/040-zookeeper.sh

@@ -0,0 +1 @@
+../common/zookeeper.sh

tools/deployment/osh-infra-kafka/050-kafka.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2019 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make kafka
+
+#NOTE: Deploy command
+helm upgrade --install kafka ./kafka \
+    --namespace=osh-infra \
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh osh-infra
+
+#NOTE: Validate deployment info
+helm status kafka
+
+#NOTE: Test deployment
+helm test kafka

zuul.d/jobs.yaml

@@ -137,6 +137,26 @@
         - ./tools/deployment/osh-infra-logging/075-kibana.sh
         - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true
 
+- job:
+    name: openstack-helm-infra-kafka
+    parent: openstack-helm-infra-functional
+    timeout: 7200
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+    run: playbooks/osh-infra-gate-runner.yaml
+    post-run: playbooks/osh-infra-collect-logs.yaml
+    nodeset: openstack-helm-single-node
+    vars:
+      gate_scripts:
+        - ./tools/deployment/osh-infra-kafka/000-install-packages.sh
+        - ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh
+        - ./tools/deployment/osh-infra-kafka/010-ingress.sh
+        - ./tools/deployment/osh-infra-kafka/020-ceph.sh
+        - ./tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh
+        - ./tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh
+        - ./tools/deployment/osh-infra-kafka/040-zookeeper.sh
+        - ./tools/deployment/osh-infra-kafka/050-kafka.sh
+
 - job:
     name: openstack-helm-infra-aio-monitoring
     parent: openstack-helm-infra-functional

zuul.d/project.yaml

@@ -23,6 +23,8 @@
         - openstack-helm-infra-aio-monitoring
         - openstack-helm-infra-federated-monitoring:
             voting: false
+        - openstack-helm-infra-kafka:
+            voting: false
         - openstack-helm-infra-aio-network-policy:
             voting: false
         - openstack-helm-infra-openstack-support