From 558a4b5b2c4adcfacf18b41632a4e3cc6d3487ff Mon Sep 17 00:00:00 2001 From: Steven Fitzpatrick Date: Fri, 19 Feb 2021 22:13:17 +0000 Subject: [PATCH] [RFC] Remove Kafka and Zookeeper from osh-infra These charts have not been used or maintained in quite a while Change-Id: Ia5778a7a7e74e4938d5f0a1cf17102fc2b5cc779 --- kafka/Chart.yaml | 26 -- kafka/requirements.yaml | 18 - kafka/templates/bin/_generate-acl.sh.tpl | 50 --- kafka/templates/bin/_helm-test.sh.tpl | 122 ------ kafka/templates/bin/_kafka-probe.sh.tpl | 19 - kafka/templates/bin/_kafka.sh.tpl | 35 -- kafka/templates/configmap-bin.yaml | 33 -- kafka/templates/configmap-etc.yaml | 25 -- kafka/templates/ingress-kafka.yaml | 18 - kafka/templates/job-generate-acl.yaml | 72 ---- kafka/templates/job-image-repo-sync.yaml | 18 - .../prometheus/bin/_kafka-exporter.sh.tpl | 31 -- .../monitoring/prometheus/configmap-bin.yaml | 25 -- .../monitoring/prometheus/deployment.yaml | 98 ----- .../monitoring/prometheus/network-policy.yaml | 18 - .../prometheus/secret-exporter.yaml | 27 -- .../monitoring/prometheus/service.yaml | 36 -- kafka/templates/network_policy.yaml | 17 - kafka/templates/pod-helm-test.yaml | 74 ---- kafka/templates/secret-ingress-tls.yaml | 17 - kafka/templates/secret-kafka.yaml | 27 -- kafka/templates/service-discovery.yaml | 32 -- kafka/templates/service-ingress-kafka.yaml | 18 - kafka/templates/service.yaml | 36 -- kafka/templates/statefulset.yaml | 191 --------- kafka/values.yaml | 379 ------------------ releasenotes/config.yaml | 2 - releasenotes/notes/kafka.yaml | 5 - releasenotes/notes/zookeeper.yaml | 5 - .../osh-infra-kafka/000-install-packages.sh | 1 - .../osh-infra-kafka/005-deploy-k8s.sh | 1 - .../deployment/osh-infra-kafka/010-ingress.sh | 1 - tools/deployment/osh-infra-kafka/020-ceph.sh | 1 - .../osh-infra-kafka/025-ceph-ns-activate.sh | 1 - .../osh-infra-kafka/030-radosgw-osh-infra.sh | 1 - .../osh-infra-kafka/040-zookeeper.sh | 1 - tools/deployment/osh-infra-kafka/050-kafka.sh | 33 -- zuul.d/jobs.yaml | 20 - zuul.d/project.yaml | 1 - 39 files changed, 1535 deletions(-) delete mode 100644 kafka/Chart.yaml delete mode 100644 kafka/requirements.yaml delete mode 100644 kafka/templates/bin/_generate-acl.sh.tpl delete mode 100644 kafka/templates/bin/_helm-test.sh.tpl delete mode 100644 kafka/templates/bin/_kafka-probe.sh.tpl delete mode 100644 kafka/templates/bin/_kafka.sh.tpl delete mode 100644 kafka/templates/configmap-bin.yaml delete mode 100644 kafka/templates/configmap-etc.yaml delete mode 100644 kafka/templates/ingress-kafka.yaml delete mode 100644 kafka/templates/job-generate-acl.yaml delete mode 100644 kafka/templates/job-image-repo-sync.yaml delete mode 100644 kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl delete mode 100644 kafka/templates/monitoring/prometheus/configmap-bin.yaml delete mode 100644 kafka/templates/monitoring/prometheus/deployment.yaml delete mode 100644 kafka/templates/monitoring/prometheus/network-policy.yaml delete mode 100644 kafka/templates/monitoring/prometheus/secret-exporter.yaml delete mode 100644 kafka/templates/monitoring/prometheus/service.yaml delete mode 100644 kafka/templates/network_policy.yaml delete mode 100644 kafka/templates/pod-helm-test.yaml delete mode 100644 kafka/templates/secret-ingress-tls.yaml delete mode 100644 kafka/templates/secret-kafka.yaml delete mode 100644 kafka/templates/service-discovery.yaml delete mode 100644 kafka/templates/service-ingress-kafka.yaml delete mode 100644 kafka/templates/service.yaml delete mode 100644 kafka/templates/statefulset.yaml delete mode 100644 kafka/values.yaml delete mode 100644 releasenotes/notes/kafka.yaml delete mode 100644 releasenotes/notes/zookeeper.yaml delete mode 120000 tools/deployment/osh-infra-kafka/000-install-packages.sh delete mode 120000 tools/deployment/osh-infra-kafka/005-deploy-k8s.sh delete mode 120000 tools/deployment/osh-infra-kafka/010-ingress.sh delete mode 120000 tools/deployment/osh-infra-kafka/020-ceph.sh delete mode 120000 tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh delete mode 120000 tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh delete mode 120000 tools/deployment/osh-infra-kafka/040-zookeeper.sh delete mode 100755 tools/deployment/osh-infra-kafka/050-kafka.sh diff --git a/kafka/Chart.yaml b/kafka/Chart.yaml deleted file mode 100644 index 47d71aed9..000000000 --- a/kafka/Chart.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -apiVersion: v1 -appVersion: v2.12-2.3.0 -description: OpenStack-Helm Kafka -name: kafka -version: 0.1.1 -home: https://kafka.apache.org/ -sources: - - https://github.com/apache/kafka - - https://github.com/danielqsj/kafka_exporter - - https://opendev.org/openstack/openstack-helm-infra -maintainers: - - name: OpenStack-Helm Authors -... diff --git a/kafka/requirements.yaml b/kafka/requirements.yaml deleted file mode 100644 index 19b0d6992..000000000 --- a/kafka/requirements.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -dependencies: - - name: helm-toolkit - repository: http://localhost:8879/charts - version: ">= 0.1.0" -... diff --git a/kafka/templates/bin/_generate-acl.sh.tpl b/kafka/templates/bin/_generate-acl.sh.tpl deleted file mode 100644 index 1d1530851..000000000 --- a/kafka/templates/bin/_generate-acl.sh.tpl +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */}} - -{{- $envAll := . }} - -{{- if .Values.monitoring.prometheus.enabled }} -{{- $credentials := .Values.endpoints.kafka_exporter.auth }} -/opt/kafka/bin/kafka-acls.sh \ - --authorizer kafka.security.auth.SimpleAclAuthorizer \ - --authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \ - --add \ - --allow-principal User:{{ $credentials.username }} \ - --operation DESCRIBE \ - --topic "*" \ - --group "*" \ - --cluster -{{ end }} - -{{ $producers := .Values.conf.kafka.jaas.producers }} -{{- range $producer, $properties := $producers }} -/opt/kafka/bin/kafka-acls.sh \ - --authorizer kafka.security.auth.SimpleAclAuthorizer \ - --authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \ - --add \ - --allow-principal User:{{ $properties.username }} \ - --producer \ - --topic {{ $properties.topic | quote }} -{{- end }} - -{{ $consumers := .Values.conf.kafka.jaas.consumers }} -{{- range $consumer, $properties := $consumers }} -/opt/kafka/bin/kafka-acls.sh \ - --authorizer kafka.security.auth.SimpleAclAuthorizer \ - --authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \ - --add \ - --allow-principal User:{{ $properties.username }} \ - --consumer \ - --topic {{ $properties.topic | quote }} \ - --group {{ $properties.group | quote }} -{{- end }} diff --git a/kafka/templates/bin/_helm-test.sh.tpl b/kafka/templates/bin/_helm-test.sh.tpl deleted file mode 100644 index 979d20937..000000000 --- a/kafka/templates/bin/_helm-test.sh.tpl +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -function create_topic () { - ./opt/kafka/bin/kafka-topics.sh \ - --create --topic $1 \ - --partitions $2 \ - --replication-factor $3 \ - --zookeeper $KAFKA_ZOOKEEPER_CONNECT -} - -function describe_topic () { - ./opt/kafka/bin/kafka-topics.sh \ - --describe --topic $1 \ - --zookeeper $KAFKA_ZOOKEEPER_CONNECT -} - -function produce_message () { - echo $2 | \ - ./opt/kafka/bin/kafka-console-producer.sh \ - --topic $1 \ - --broker-list $KAFKA_BROKERS -} - -function consume_messages () { - ./opt/kafka/bin/kafka-console-consumer.sh \ - --topic $1 \ - --timeout-ms 5000 \ - --from-beginning \ - --bootstrap-server $KAFKA_BROKERS -} - -function delete_partition_messages () { - ./opt/kafka/bin/kafka-delete-records.sh \ - --offset-json-file $1 \ - --bootstrap-server $KAFKA_BROKERS -} - -function delete_topic () { - ./opt/kafka/bin/kafka-topics.sh \ - --delete --topic $1 \ - --zookeeper $KAFKA_ZOOKEEPER_CONNECT -} - -set -ex - -TOPIC="kafka-test" -PARTITION_COUNT=3 -PARTITION_REPLICAS=2 - -echo "Creating topic $TOPIC" -create_topic $TOPIC $PARTITION_COUNT $PARTITION_REPLICAS -describe_topic $TOPIC - -# Note: The commands used here are not playing well with the WIP -# SASL auth implementation. Commenting for now: - -# echo "Producing 5 messages" -# for i in {1..5}; do -# MESSAGE="Message #$i" -# produce_message $TOPIC "$MESSAGE" -# done - -# echo -e "\nConsuming messages (A \"TimeoutException\" is expected, else this would consume forever)" -# consume_messages $TOPIC - -# echo "Producing 5 more messages" -# for i in {6..10}; do -# MESSAGE="Message #$i" -# produce_message $TOPIC "$MESSAGE" -# done - -# echo -e "\nCreating partition offset reset json file" -# tee /tmp/partition_offsets.json << EOF -# { -# "partitions": [ -# { -# "topic": "$TOPIC", -# "partition": 0, -# "offset": -1 -# }, { -# "topic": "$TOPIC", -# "partition": 1, -# "offset": -1 -# }, { -# "topic": "$TOPIC", -# "partition": 2, -# "offset": -1 -# } -# ], -# "version": 1 -# } -# EOF - -# echo "Resetting $TOPIC partitions (deleting messages)" -# delete_partition_messages /tmp/partition_offsets.json - -echo "Deleting topic $TOPIC" -delete_topic $TOPIC >> /tmp/deletion - -cat /tmp/deletion - -if [ $(cat /tmp/deletion | grep 'marked for deletion' | wc -l) -eq 1 ] -then - echo "Topic $TOPIC was deleted successfully." - exit 0 -else - echo "Topic $TOPIC was not successfully deleted." - exit 1 -fi diff --git a/kafka/templates/bin/_kafka-probe.sh.tpl b/kafka/templates/bin/_kafka-probe.sh.tpl deleted file mode 100644 index b46c0d1c4..000000000 --- a/kafka/templates/bin/_kafka-probe.sh.tpl +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -ex - -echo ruok | nc 127.0.0.1 ${KAFKA_PORT} diff --git a/kafka/templates/bin/_kafka.sh.tpl b/kafka/templates/bin/_kafka.sh.tpl deleted file mode 100644 index e567cb833..000000000 --- a/kafka/templates/bin/_kafka.sh.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if not (empty .Values.conf.kafka.server_settings) }} -{{- range $key, $value := .Values.conf.kafka.server_settings }} -{{- $varName := printf "%s%s" "KAFKA_" ($key | upper) }} -{{- $varValue := ternary ($value | quote) ($value | int) (kindIs "string" $value) }} -export {{ $varName }}={{ $varValue }} -{{- end }} -{{- end }} -export KAFKA_SUPER_USERS="User:$ADMIN_USERNAME" - -COMMAND="${@:-start}" - -function start() { - ./usr/bin/start-kafka.sh -} - -function stop () { - kill -TERM 1 -} - -$COMMAND diff --git a/kafka/templates/configmap-bin.yaml b/kafka/templates/configmap-bin.yaml deleted file mode 100644 index 3fe398ea3..000000000 --- a/kafka/templates/configmap-bin.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.configmap_bin }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kafka-bin -data: - kafka.sh: | -{{ tuple "bin/_kafka.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - kafka-liveness.sh: | -{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - kafka-readiness.sh: | -{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - helm-test.sh: | -{{ tuple "bin/_helm-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - generate-acl.sh: | -{{ tuple "bin/_generate-acl.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end -}} \ No newline at end of file diff --git a/kafka/templates/configmap-etc.yaml b/kafka/templates/configmap-etc.yaml deleted file mode 100644 index 515bddc53..000000000 --- a/kafka/templates/configmap-etc.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.configmap_etc }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: kafka-etc -type: Opaque -data: -{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.kafka.jaas.template "key" "jaas.conf" "format" "Secret") | indent 2 }} -{{- end }} diff --git a/kafka/templates/ingress-kafka.yaml b/kafka/templates/ingress-kafka.yaml deleted file mode 100644 index c453a40bc..000000000 --- a/kafka/templates/ingress-kafka.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.ingress .Values.network.kafka.ingress.public }} -{{- $ingressOpts := dict "envAll" . "backendService" "kafka" "backendServiceType" "kafka" "backendPort" "broker" -}} -{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} -{{- end }} diff --git a/kafka/templates/job-generate-acl.yaml b/kafka/templates/job-generate-acl.yaml deleted file mode 100644 index 6a3088bc9..000000000 --- a/kafka/templates/job-generate-acl.yaml +++ /dev/null @@ -1,72 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.job_generate_acl }} -{{- $envAll := . }} - -{{- $KafkaUserSecret := .Values.secrets.kafka.admin }} - -{{- $serviceAccountName := "kafka-generate-acl" }} -{{ tuple $envAll "generate_acl" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: kafka-generate-acl - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: - backoffLimit: {{ .Values.jobs.generate_acl.backoffLimit }} - template: - metadata: - labels: -{{ tuple $envAll "kafka" "generate-acl" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - spec: -{{ dict "envAll" $envAll "application" "generate-acl" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} - serviceAccountName: {{ $serviceAccountName }} - activeDeadlineSeconds: {{ .Values.jobs.generate_acl.activeDeadlineSeconds }} - restartPolicy: OnFailure - nodeSelector: - {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} - initContainers: -{{ tuple $envAll "generate_acl" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: generate-acl -{{ tuple $envAll "generate_acl" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.generate_acl | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "generate_acl" "container" "generate_acl" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - env: - - name: KAFKA_ZOOKEEPER_CONNECT - value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" - command: - - /tmp/generate-acl.sh - volumeMounts: - - name: kafka-bin - mountPath: /tmp/generate-acl.sh - subPath: generate-acl.sh - readOnly: true - - name: kafka-etc - mountPath: /opt/kafka/config/jaas.conf - subPath: jaas.conf - readOnly: true - volumes: - - name: kafka-bin - configMap: - name: kafka-bin - defaultMode: 0555 - - name: kafka-etc - secret: - secretName: kafka-etc - defaultMode: 0444 -{{- end }} diff --git a/kafka/templates/job-image-repo-sync.yaml b/kafka/templates/job-image-repo-sync.yaml deleted file mode 100644 index 787859bda..000000000 --- a/kafka/templates/job-image-repo-sync.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} -{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kafka" -}} -{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} -{{- end }} diff --git a/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl b/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl deleted file mode 100644 index 86c66eb59..000000000 --- a/kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -COMMAND="${@:-start}" - -function start () { - exec /bin/kafka_exporter \ - --sasl.enabled \ - --sasl.username=$KAFKA_EXPORTER_USERNAME \ - --sasl.password=$KAFKA_EXPORTER_PASSWORD \ - --kafka.server=$KAFKA_BROKERS -} - -function stop () { - kill -TERM 1 -} - -$COMMAND diff --git a/kafka/templates/monitoring/prometheus/configmap-bin.yaml b/kafka/templates/monitoring/prometheus/configmap-bin.yaml deleted file mode 100644 index ac8b6b589..000000000 --- a/kafka/templates/monitoring/prometheus/configmap-bin.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kafka-exporter-bin -data: - kafka-exporter.sh: | -{{ tuple "bin/_kafka-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} diff --git a/kafka/templates/monitoring/prometheus/deployment.yaml b/kafka/templates/monitoring/prometheus/deployment.yaml deleted file mode 100644 index ae6f0fc73..000000000 --- a/kafka/templates/monitoring/prometheus/deployment.yaml +++ /dev/null @@ -1,98 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.monitoring.prometheus.deployment .Values.monitoring.prometheus.enabled }} -{{- $envAll := . }} - -{{- $kafkaExporterUserSecret := .Values.secrets.kafka_exporter.user }} - -{{- $serviceAccountName := "prometheus-kafka-exporter" }} -{{ tuple $envAll "kafka_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus-kafka-exporter - labels: -{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - replicas: {{ .Values.pod.replicas.kafka_exporter }} - selector: - matchLabels: -{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} -{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} - template: - metadata: - labels: -{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: -{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - spec: -{{ dict "envAll" $envAll "application" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }} - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka_exporter.timeout | default "30" }} - initContainers: -{{ tuple $envAll "kafka_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: kafka-exporter -{{ tuple $envAll "kafka_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.kafka_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "kafka_exporter" "container" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - command: - - /tmp/kafka-exporter.sh - - start - lifecycle: - preStop: - exec: - command: - - /tmp/kafka-exporter.sh - - stop - env: - - name: KAFKA_BROKERS - value: {{ tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | quote }} - - name: KAFKA_EXPORTER_USERNAME - valueFrom: - secretKeyRef: - name: {{ $kafkaExporterUserSecret }} - key: KAFKA_EXPORTER_USERNAME - - name: KAFKA_EXPORTER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ $kafkaExporterUserSecret }} - key: KAFKA_EXPORTER_PASSWORD - ports: - - name: exporter - containerPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - readinessProbe: - tcpSocket: - port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - initialDelaySeconds: 20 - periodSeconds: 10 - volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: kafka-exporter-bin - mountPath: /tmp/kafka-exporter.sh - subPath: kafka-exporter.sh - readOnly: true - volumes: - - name: pod-tmp - emptyDir: {} - - name: kafka-exporter-bin - configMap: - name: kafka-exporter-bin - defaultMode: 0555 -{{- end }} diff --git a/kafka/templates/monitoring/prometheus/network-policy.yaml b/kafka/templates/monitoring/prometheus/network-policy.yaml deleted file mode 100644 index ed8f72abe..000000000 --- a/kafka/templates/monitoring/prometheus/network-policy.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.monitoring.prometheus.network_policy .Values.monitoring.prometheus.enabled -}} -{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-kafka-exporter" -}} -{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} -{{- end -}} diff --git a/kafka/templates/monitoring/prometheus/secret-exporter.yaml b/kafka/templates/monitoring/prometheus/secret-exporter.yaml deleted file mode 100644 index e6946ae31..000000000 --- a/kafka/templates/monitoring/prometheus/secret-exporter.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.secret_kafka }} -{{- $envAll := . }} -{{- $secretName := .Values.secrets.kafka_exporter.user }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secretName }} -type: Opaque -data: - KAFKA_EXPORTER_USERNAME: {{ .Values.endpoints.kafka_exporter.auth.username | b64enc }} - KAFKA_EXPORTER_PASSWORD: {{ .Values.endpoints.kafka_exporter.auth.password | b64enc }} -{{- end }} diff --git a/kafka/templates/monitoring/prometheus/service.yaml b/kafka/templates/monitoring/prometheus/service.yaml deleted file mode 100644 index c2a5a7227..000000000 --- a/kafka/templates/monitoring/prometheus/service.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.monitoring.prometheus.service .Values.monitoring.prometheus.enabled }} -{{- $envAll := . }} -{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kafka_exporter }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "kafka_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - labels: -{{ tuple $envAll "kafka-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - annotations: -{{- if .Values.monitoring.prometheus.enabled }} -{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} -{{- end }} -spec: - ports: - - name: exporter - port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - targetPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - selector: -{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -{{- end }} diff --git a/kafka/templates/network_policy.yaml b/kafka/templates/network_policy.yaml deleted file mode 100644 index ebbd91608..000000000 --- a/kafka/templates/network_policy.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */}} - -{{- if .Values.manifests.network_policy -}} -{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kafka" -}} -{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} -{{- end -}} diff --git a/kafka/templates/pod-helm-test.yaml b/kafka/templates/pod-helm-test.yaml deleted file mode 100644 index 0a84066d6..000000000 --- a/kafka/templates/pod-helm-test.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.helm_test }} -{{- $envAll := . }} - -{{- $serviceAccountName := print .Release.Name "-test" }} -{{ tuple $envAll "test" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: v1 -kind: Pod -metadata: - name: "{{.Release.Name}}-test" - labels: -{{ tuple $envAll "kafka" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - annotations: - "helm.sh/hook": test-success - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} -spec: -{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} - serviceAccountName: {{ $serviceAccountName }} - nodeSelector: - {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} - restartPolicy: Never - initContainers: -{{ tuple $envAll "test" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} - containers: - - name: {{.Release.Name}}-helm-test -{{ tuple $envAll "helm_test" | include "helm-toolkit.snippets.image" | indent 6 }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.test | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} -{{ dict "envAll" $envAll "application" "test" "container" "helm_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} - command: - - "/tmp/helm-test.sh" - env: - - name: KAFKA_ZOOKEEPER_CONNECT - value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" - - name: KAFKA_BROKERS - value: "{{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" - - name: KAFKA_OPTS - value: {{ include "helm-toolkit.utils.joinListWithSpace" .Values.conf.kafka.jvm_options | quote }} - volumeMounts: - - name: pod-tmp - mountPath: /tmp - - name: kafka-bin - mountPath: /tmp/helm-test.sh - subPath: helm-test.sh - readOnly: true - - name: kafka-etc - mountPath: /opt/kafka/config/jaas.conf - subPath: jaas.conf - readOnly: true - volumes: - - name: pod-tmp - emptyDir: {} - - name: kafka-bin - configMap: - name: kafka-bin - defaultMode: 0555 - - name: kafka-etc - secret: - secretName: kafka-etc - defaultMode: 0444 -{{- end }} diff --git a/kafka/templates/secret-ingress-tls.yaml b/kafka/templates/secret-ingress-tls.yaml deleted file mode 100644 index afe2c6526..000000000 --- a/kafka/templates/secret-ingress-tls.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.secret_ingress_tls -}} -{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "kafka" "backendService" "kafka" ) }} -{{- end }} diff --git a/kafka/templates/secret-kafka.yaml b/kafka/templates/secret-kafka.yaml deleted file mode 100644 index a4eaac600..000000000 --- a/kafka/templates/secret-kafka.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.secret_kafka }} -{{- $envAll := . }} -{{- $secretName := .Values.secrets.kafka.admin }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secretName }} -type: Opaque -data: - KAFKA_ADMIN_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }} - KAFKA_ADMIN_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }} -{{- end }} diff --git a/kafka/templates/service-discovery.yaml b/kafka/templates/service-discovery.yaml deleted file mode 100644 index 139cfc7cc..000000000 --- a/kafka/templates/service-discovery.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service_discovery }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "kafka" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - labels: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - ports: - - name: broker - targetPort: broker - port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - clusterIP: None - selector: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -{{- end }} diff --git a/kafka/templates/service-ingress-kafka.yaml b/kafka/templates/service-ingress-kafka.yaml deleted file mode 100644 index 0a2ce8928..000000000 --- a/kafka/templates/service-ingress-kafka.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if and .Values.manifests.service_ingress .Values.network.kafka.ingress.public }} -{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kafka" -}} -{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} -{{- end }} diff --git a/kafka/templates/service.yaml b/kafka/templates/service.yaml deleted file mode 100644 index a68814b90..000000000 --- a/kafka/templates/service.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.service }} -{{- $envAll := . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ tuple "kafka" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - labels: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - ports: - - name: broker - port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - {{ if .Values.network.kafka.node_port.enabled }} - nodePort: {{ .Values.network.kafka.node_port.port }} - {{ end }} - selector: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} - {{ if .Values.network.kafka.node_port.enabled }} - type: NodePort - {{ end }} -{{- end }} diff --git a/kafka/templates/statefulset.yaml b/kafka/templates/statefulset.yaml deleted file mode 100644 index 0b3390b35..000000000 --- a/kafka/templates/statefulset.yaml +++ /dev/null @@ -1,191 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.statefulset }} -{{- $envAll := . }} - -{{- $mounts_kafka := .Values.pod.mounts.kafka.kafka }} -{{- $mounts_kafka_init := .Values.pod.mounts.kafka.init_container }} -{{- $kafkaUserSecret := .Values.secrets.kafka.admin }} -{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - -{{- $serviceAccountName := printf "%s-%s" .Release.Name "kafka" }} -{{ tuple $envAll "kafka" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ $serviceAccountName }} -rules: - - apiGroups: - - "" - resources: - - nodes - - nodes/proxy - - services - - endpoints - - pods - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ $serviceAccountName }} -subjects: - - kind: ServiceAccount - name: {{ $serviceAccountName }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ $serviceAccountName }} - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: kafka - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} - labels: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - serviceName: {{ tuple "kafka" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - replicas: {{ .Values.pod.replicas.kafka }} - updateStrategy: - type: OnDelete - podManagementPolicy: Parallel - selector: - matchLabels: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} - template: - metadata: - labels: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: -{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} - spec: -{{ dict "envAll" $envAll "application" "kafka" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} - serviceAccountName: {{ $serviceAccountName }} - affinity: -{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} - nodeSelector: - {{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }} - terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka.timeout | default "30" }} - initContainers: -{{ tuple $envAll "kafka" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: kafka - command: - - "/tmp/kafka.sh" -{{ tuple $envAll "kafka" | include "helm-toolkit.snippets.image" | indent 10 }} -{{ tuple $envAll $envAll.Values.pod.resources.kafka | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} -{{ dict "envAll" $envAll "application" "kafka" "container" "kafka" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} - ports: - - name: broker - containerPort: {{ $kafkaBrokerPort }} - env: - - name: ADMIN_USERNAME - valueFrom: - secretKeyRef: - name: {{ $kafkaUserSecret }} - key: KAFKA_ADMIN_USERNAME - - name: KAFKA_PORT - value: "{{ $kafkaBrokerPort }}" - - name: ZOOKEEPER_PORT - value: "{{ tuple "zookeeper" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" - - name: KAFKA_ZOOKEEPER_CONNECT - value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}" - - name: KAFKA_LISTENERS - value: "PLAINTEXT://:{{$kafkaBrokerPort}}" - - name: KAFKA_CREATE_TOPICS - value: "{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.kafka.topics }}" - - name: KAFKA_OPTS - value: {{ include "helm-toolkit.utils.joinListWithSpace" .Values.conf.kafka.jvm_options | quote }} - readinessProbe: - initialDelaySeconds: 20 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 2 - successThreshold: 1 - exec: - command: - - /tmp/kafka-readiness.sh - livenessProbe: - initialDelaySeconds: 20 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 2 - successThreshold: 1 - exec: - command: - - /tmp/kafka-liveness.sh - volumeMounts: - - name: kafka-bin - mountPath: /tmp/kafka.sh - subPath: kafka.sh - readOnly: true - - name: kafka-bin - mountPath: /tmp/kafka-liveness.sh - subPath: kafka-liveness.sh - readOnly: true - - name: kafka-bin - mountPath: /tmp/kafka-readiness.sh - subPath: kafka-readiness.sh - readOnly: true - - name: kafka-etc - mountPath: /opt/kafka/config/jaas.conf - subPath: jaas.conf - readOnly: true - - name: data - mountPath: {{ .Values.conf.kafka.config.data_directory }} -{{ if $mounts_kafka.volumeMounts }}{{ toYaml $mounts_kafka.volumeMounts | indent 12 }}{{ end }} - volumes: - - name: kafka-bin - configMap: - name: kafka-bin - defaultMode: 0555 - - name: kafka-etc - secret: - secretName: kafka-etc - defaultMode: 0444 -{{ if $mounts_kafka.volumes }}{{ toYaml $mounts_kafka.volumes | indent 8 }}{{ end }} -{{- if not .Values.storage.enabled }} - - name: data - emptyDir: {} -{{- else }} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: {{ .Values.storage.pvc.access_mode }} - resources: - requests: - storage: {{ .Values.storage.requests.storage }} - storageClassName: {{ .Values.storage.storage_class }} -{{- end }} -{{- end }} diff --git a/kafka/values.yaml b/kafka/values.yaml deleted file mode 100644 index 239675dc2..000000000 --- a/kafka/values.yaml +++ /dev/null @@ -1,379 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for kafka. -# This is a YAML-formatted file. -# Declare name/value pairs to be passed into your templates. -# name: value - ---- -images: - tags: - kafka: docker.io/wurstmeister/kafka:2.12-2.3.0 - kafka_exporter: docker.io/danielqsj/kafka-exporter:latest - dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 - image_repo_sync: docker.io/docker:17.07.0 - helm_test: docker.io/wurstmeister/kafka:2.12-2.3.0 - generate_acl: docker.io/wurstmeister/kafka:2.12-2.3.0 - pull_policy: IfNotPresent - local_registry: - active: false - exclude: - - dep_check - - image_repo_sync - -labels: - kafka: - node_selector_key: openstack-control-plane - node_selector_value: enabled - job: - node_selector_key: openstack-control-plane - node_selector_value: enabled - test: - node_selector_key: openstack-control-plane - node_selector_value: enabled - -pod: - security_context: - kafka: - pod: {} - container: - kafka: {} - kafka-init: {} - kafka_exporter: - pod: {} - container: - kafka_exporter: {} - generate_acl: - pod: {} - container: - generate_acl: {} - affinity: - anti: - type: - default: preferredDuringSchedulingIgnoredDuringExecution - topologyKey: - default: kubernetes.io/hostname - weight: - default: 10 - mounts: - kafka: - kafka: - init_container: null - replicas: - kafka: 3 - kafka_exporter: 1 - lifecycle: - upgrades: - statefulsets: - pod_replacement_strategy: RollingUpdate - termination_grace_period: - kafka: - timeout: 30 - kafka_exporter: - timeout: 30 - resources: - enabled: false - kafka: - limits: - memory: "1024Mi" - cpu: "2000m" - requests: - memory: "128Mi" - cpu: "500m" - kafka_exporter: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - jobs: - image_repo_sync: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - generate_acl: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - test: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "1024Mi" - cpu: "2000m" - -endpoints: - cluster_domain_suffix: cluster.local - local_image_registry: - name: docker-registry - namespace: docker-registry - hosts: - default: localhost - internal: docker-registry - node: localhost - host_fqdn_override: - default: null - port: - registry: - node: 5000 - kafka: - name: kafka - namespace: null - auth: - admin: - username: admin - password: changeme - hosts: - default: kafka-broker - discovery: kafka-discovery - public: kafka - host_fqdn_override: - default: null - # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public - # endpoints using the following format: - # public: - # host: null - # tls: - # crt: null - # key: null - path: - default: null - scheme: - default: 'http' - port: - broker: - default: 9092 - kafka-exporter: - default: 9141 - jmx-exporter: - default: 9404 - kafka_exporter: - auth: - username: kafka-exporter - password: changeme - namespace: null - hosts: - default: kafka-exporter - host_fqdn_override: - default: null - scheme: - default: 'http' - port: - exporter: - default: 9308 - zookeeper: - name: zookeeper - namespace: null - auth: - admin: - username: admin - password: changeme - hosts: - default: zookeeper-int - public: zookeeper - host_fqdn_override: - default: null - path: - default: null - scheme: - default: 'http' - port: - client: - default: 2181 - server: - default: 2888 - -dependencies: - dynamic: - common: - local_image_registry: - jobs: - - kafka-image-repo-sync - services: - - endpoint: node - service: local_image_registry - static: - image_repo_sync: - services: - - endpoint: internal - service: local_image_registry - kafka: - services: - - endpoint: internal - service: zookeeper - kafka_exporter: - services: - - endpoint: internal - service: kafka - generate_acl: - services: - - endpoint: internal - service: kafka - -monitoring: - prometheus: - enabled: true - kafka_exporter: - scrape: true - -network: - kafka: - ingress: - public: true - classes: - namespace: "nginx" - cluster: "nginx-cluster" - annotations: - nginx.ingress.kubernetes.io/rewrite-target: / - nginx.ingress.kubernetes.io/affinity: cookie - nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-kafka - nginx.ingress.kubernetes.io/session-cookie-hash: sha1 - nginx.ingress.kubernetes.io/session-cookie-expires: "600" - nginx.ingress.kubernetes.io/session-cookie-max-age: "600" - node_port: - enabled: false - port: 31033 - -network_policy: - kafka: - ingress: - - {} - egress: - - {} - kafka_exporter: - ingress: - - {} - egress: - - {} - -secrets: - tls: - kafka: - kafka: - public: kafka-tls-public - kafka: - admin: kafka-admin-creds - kafka_exporter: - user: kafka-exporter-creds - -storage: - enabled: true - pvc: - name: kafka-pvc - access_mode: ["ReadWriteOnce"] - requests: - storage: 5Gi - storage_class: general - -manifests: - configmap_bin: true - configmap_etc: true - helm_test: true - ingress: true - job_image_repo_sync: true - job_generate_acl: true - monitoring: - prometheus: - configmap_bin: true - deployment: true - secret_exporter: true - service: true - network_policy: false - network_policy: false - secret_ingress_tls: true - secret_kafka: true - secret_zookeeper: true - service_discovery: true - service_ingress: true - service: true - statefulset: true - -jobs: - generate_acl: - backoffLimit: 6 - activeDeadlineSeconds: 600 - -conf: - kafka: - config: - data_directory: /var/lib/kafka/data - server_settings: - # Optionally provide configuration overrides for Kafka's - # server.properties file. Replace '.' with '_' ie: - # for message.max.bytes enter message_max_bytes - message_max_bytes: 5000000 - authorizer_class_name: kafka.security.auth.SimpleAclAuthorizer - listeners: SASL_PLAINTEXT://:9092 - security_protocol: SASL_PLAINTEXT - security_inter_broker_protocol: SASL_PLAINTEXT - sasl_mechanism: PLAIN - sasl_enabled_mechanisms: PLAIN - sasl_mechanism_inter_broker_protocol: PLAIN - topics: - # List of topic strings formatted like: - # topic_name:number_of_partitions:replication_factor - # - "mytopic:1:1" - jaas: # Define Authentication Details in this section - producers: - # region_a: # Just an ID used to iterate through the dict of producers - # username: region-a-producer - # password: changeme - # topic: region-a # Used in generate-acl.sh to provide access - consumers: - # region_a: # Just an ID used to iterate through the dict of consumers - # username: region-a-consumer - # password: changeme - # topic: region-a # Used in generate-acl.sh to provide access - # group: region-a # Used in generate-acl.sh to provide access - template: | - KafkaServer { - org.apache.kafka.common.security.plain.PlainLoginModule required - {{- $admin := .Values.endpoints.kafka.auth.admin }} - username={{ $admin.username | quote}} - password={{ $admin.password | quote}} - user_{{ $admin.username }}={{ $admin.password | quote }} - {{- if .Values.monitoring.prometheus.enabled }} - {{- $exporter := .Values.endpoints.kafka_exporter.auth }} - user_{{ $exporter.username }}={{ $exporter.password | quote }} - {{- end }} - {{- range $producer, $credentials := .Values.conf.kafka.jaas.producers }} - user_{{ $credentials.username }}={{ $credentials.password | quote }} - {{- end }} - {{- range $consumer, $credentials := .Values.conf.kafka.jaas.producers }} - user_{{ $credentials.username }}={{ $credentials.password | quote }} - {{- end }} - {{- printf ";" }} - }; - KafkaClient { - org.apache.kafka.common.security.plain.PlainLoginModule required - username={{ $admin.username | quote}} - password={{ $admin.password | quote}} - {{- printf ";" }} - }; - Client { - org.apache.kafka.common.security.plain.PlainLoginModule required - username={{ $admin.username | quote}} - password={{ $admin.password | quote}} - {{- printf ";" }} - }; - jvm_options: - - -Djava.security.auth.login.config=/opt/kafka/config/jaas.conf -... diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml index ca1691502..8ac806d6a 100644 --- a/releasenotes/config.yaml +++ b/releasenotes/config.yaml @@ -24,7 +24,6 @@ sections: - [grafana, grafana Chart] - [helm-toolkit, helm-toolkit Chart] - [ingress, ingress Chart] - - [kafka, kafka Chart] - [kibana, kibana Chart] - [kube-dns, kube-dns Chart] - [kubernetes-keystone-webhook, kubernetes-keystone-webhook Chart] @@ -55,7 +54,6 @@ sections: - [redis, redis Chart] - [registry, registry Chart] - [tiller, tiller Chart] - - [zookeeper, zookeeper Chart] - [features, New Features] - [issues, Known Issues] - [upgrade, Upgrade Notes] diff --git a/releasenotes/notes/kafka.yaml b/releasenotes/notes/kafka.yaml deleted file mode 100644 index 90709b141..000000000 --- a/releasenotes/notes/kafka.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -kafka: - - 0.1.0 Initial Chart - - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" -... diff --git a/releasenotes/notes/zookeeper.yaml b/releasenotes/notes/zookeeper.yaml deleted file mode 100644 index 866ae971b..000000000 --- a/releasenotes/notes/zookeeper.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -zookeeper: - - 0.1.0 Initial Chart - - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" -... diff --git a/tools/deployment/osh-infra-kafka/000-install-packages.sh b/tools/deployment/osh-infra-kafka/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/osh-infra-kafka/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh b/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh deleted file mode 120000 index 257a39f7a..000000000 --- a/tools/deployment/osh-infra-kafka/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../common/005-deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/010-ingress.sh b/tools/deployment/osh-infra-kafka/010-ingress.sh deleted file mode 120000 index 4c3d424df..000000000 --- a/tools/deployment/osh-infra-kafka/010-ingress.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/010-ingress.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/020-ceph.sh b/tools/deployment/osh-infra-kafka/020-ceph.sh deleted file mode 120000 index 1ab828eed..000000000 --- a/tools/deployment/osh-infra-kafka/020-ceph.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/020-ceph.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh deleted file mode 120000 index 10e71eedb..000000000 --- a/tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/025-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh b/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh deleted file mode 120000 index 1ca42d153..000000000 --- a/tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/030-radosgw-osh-infra.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/040-zookeeper.sh b/tools/deployment/osh-infra-kafka/040-zookeeper.sh deleted file mode 120000 index 69bcd4139..000000000 --- a/tools/deployment/osh-infra-kafka/040-zookeeper.sh +++ /dev/null @@ -1 +0,0 @@ -../common/zookeeper.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-kafka/050-kafka.sh b/tools/deployment/osh-infra-kafka/050-kafka.sh deleted file mode 100755 index 765a5d2c9..000000000 --- a/tools/deployment/osh-infra-kafka/050-kafka.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make kafka - -#NOTE: Deploy command -helm upgrade --install kafka ./kafka \ - --namespace=osh-infra \ - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -#NOTE: Validate deployment info -helm status kafka - -# Delete the test pod if it still exists -kubectl delete pods -l application=kafka,release_group=kafka,component=test --namespace=osh-infra --ignore-not-found -#NOTE: Test deployment -helm test kafka diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 5859e1cf1..9070c19e1 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -158,26 +158,6 @@ - ./tools/deployment/osh-infra-logging/070-kibana.sh - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true -- job: - name: openstack-helm-infra-kafka - parent: openstack-helm-infra-functional - timeout: 7200 - pre-run: - - playbooks/osh-infra-upgrade-host.yaml - post-run: playbooks/osh-infra-collect-logs.yaml - nodeset: openstack-helm-single-node - vars: - gate_scripts_relative_path: ../openstack-helm-infra - gate_scripts: - - ./tools/deployment/osh-infra-kafka/000-install-packages.sh - - ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh - - ./tools/deployment/osh-infra-kafka/010-ingress.sh - - ./tools/deployment/osh-infra-kafka/020-ceph.sh - - ./tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh - - ./tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh - - ./tools/deployment/osh-infra-kafka/040-zookeeper.sh - - ./tools/deployment/osh-infra-kafka/050-kafka.sh - - job: name: openstack-helm-infra-aio-monitoring parent: openstack-helm-infra-functional diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 52826d1dc..6fa88ca8c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -67,7 +67,6 @@ # - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-passwords - openstack-helm-infra-federated-monitoring - - openstack-helm-infra-kafka - openstack-helm-infra-local-storage - openstack-helm-infra-aio-network-policy - openstack-helm-infra-apparmor