openstack/openstack-helm-infra
Code Issues Proposed changes

[RFC] Remove Kafka and Zookeeper from osh-infra

Browse Source 
These charts have not been used or maintained in quite a while

Change-Id: Ia5778a7a7e74e4938d5f0a1cf17102fc2b5cc779
This commit is contained in:
Steven Fitzpatrick 2021-02-19 22:13:17 +00:00
parent 7ebc49c338
commit 558a4b5b2c
39 changed files with 0 additions and 1535 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
kafka/Chart.yaml
View File

@ -1,26 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: v1
appVersion: v2.12-2.3.0
description: OpenStack-Helm Kafka
name: kafka
version: 0.1.1
home: https://kafka.apache.org/
sources:
- https://github.com/apache/kafka
- https://github.com/danielqsj/kafka_exporter
- https://opendev.org/openstack/openstack-helm-infra
maintainers:
- name: OpenStack-Helm Authors
...

18
kafka/requirements.yaml
View File

@ -1,18 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
version: ">= 0.1.0"
...

50
kafka/templates/bin/_generate-acl.sh.tpl
View File

@ -1,50 +0,0 @@
#!/bin/sh
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */}}
{{- $envAll := . }}
{{- if .Values.monitoring.prometheus.enabled }}
{{- $credentials := .Values.endpoints.kafka_exporter.auth }}
/opt/kafka/bin/kafka-acls.sh \
--authorizer kafka.security.auth.SimpleAclAuthorizer \
--authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \
--add \
--allow-principal User:{{ $credentials.username }} \
--operation DESCRIBE \
--topic "*" \
--group "*" \
--cluster
{{ end }}
{{ $producers := .Values.conf.kafka.jaas.producers }}
{{- range $producer, $properties := $producers }}
/opt/kafka/bin/kafka-acls.sh \
--authorizer kafka.security.auth.SimpleAclAuthorizer \
--authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \
--add \
--allow-principal User:{{ $properties.username }} \
--producer \
--topic {{ $properties.topic | quote }}
{{- end }}
{{ $consumers := .Values.conf.kafka.jaas.consumers }}
{{- range $consumer, $properties := $consumers }}
/opt/kafka/bin/kafka-acls.sh \
--authorizer kafka.security.auth.SimpleAclAuthorizer \
--authorizer-properties zookeeper.connect=$KAFKA_ZOOKEEPER_CONNECT \
--add \
--allow-principal User:{{ $properties.username }} \
--consumer \
--topic {{ $properties.topic | quote }} \
--group {{ $properties.group | quote }}
{{- end }}

122
kafka/templates/bin/_helm-test.sh.tpl
View File

@ -1,122 +0,0 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
function create_topic () {
./opt/kafka/bin/kafka-topics.sh \
--create --topic $1 \
--partitions $2 \
--replication-factor $3 \
--zookeeper $KAFKA_ZOOKEEPER_CONNECT
}
function describe_topic () {
./opt/kafka/bin/kafka-topics.sh \
--describe --topic $1 \
--zookeeper $KAFKA_ZOOKEEPER_CONNECT
}
function produce_message () {
echo $2 | \
./opt/kafka/bin/kafka-console-producer.sh \
--topic $1 \
--broker-list $KAFKA_BROKERS
}
function consume_messages () {
./opt/kafka/bin/kafka-console-consumer.sh \
--topic $1 \
--timeout-ms 5000 \
--from-beginning \
--bootstrap-server $KAFKA_BROKERS
}
function delete_partition_messages () {
./opt/kafka/bin/kafka-delete-records.sh \
--offset-json-file $1 \
--bootstrap-server $KAFKA_BROKERS
}
function delete_topic () {
./opt/kafka/bin/kafka-topics.sh \
--delete --topic $1 \
--zookeeper $KAFKA_ZOOKEEPER_CONNECT
}
set -ex
TOPIC="kafka-test"
PARTITION_COUNT=3
PARTITION_REPLICAS=2
echo "Creating topic $TOPIC"
create_topic $TOPIC $PARTITION_COUNT $PARTITION_REPLICAS
describe_topic $TOPIC
# Note: The commands used here are not playing well with the WIP
# SASL auth implementation. Commenting for now:
# echo "Producing 5 messages"
# for i in {1..5}; do
# MESSAGE="Message #$i"
# produce_message $TOPIC "$MESSAGE"
# done
# echo -e "\nConsuming messages (A \"TimeoutException\" is expected, else this would consume forever)"
# consume_messages $TOPIC
# echo "Producing 5 more messages"
# for i in {6..10}; do
# MESSAGE="Message #$i"
# produce_message $TOPIC "$MESSAGE"
# done
# echo -e "\nCreating partition offset reset json file"
# tee /tmp/partition_offsets.json << EOF
# {
# "partitions": [
# {
# "topic": "$TOPIC",
# "partition": 0,
# "offset": -1
# }, {
# "topic": "$TOPIC",
# "partition": 1,
# "offset": -1
# }, {
# "topic": "$TOPIC",
# "partition": 2,
# "offset": -1
# }
# ],
# "version": 1
# }
# EOF
# echo "Resetting $TOPIC partitions (deleting messages)"
# delete_partition_messages /tmp/partition_offsets.json
echo "Deleting topic $TOPIC"
delete_topic $TOPIC >> /tmp/deletion
cat /tmp/deletion
if [ $(cat /tmp/deletion | grep 'marked for deletion' | wc -l) -eq 1 ]
then
echo "Topic $TOPIC was deleted successfully."
exit 0
else
echo "Topic $TOPIC was not successfully deleted."
exit 1
fi

19
kafka/templates/bin/_kafka-probe.sh.tpl
View File

@ -1,19 +0,0 @@
#!/bin/sh
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
echo ruok | nc 127.0.0.1 ${KAFKA_PORT}

35
kafka/templates/bin/_kafka.sh.tpl
View File

@ -1,35 +0,0 @@
#!/bin/bash
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if not (empty .Values.conf.kafka.server_settings) }}
{{- range $key, $value := .Values.conf.kafka.server_settings }}
{{- $varName := printf "%s%s" "KAFKA_" ($key | upper) }}
{{- $varValue := ternary ($value | quote) ($value | int) (kindIs "string" $value) }}
export {{ $varName }}={{ $varValue }}
{{- end }}
{{- end }}
export KAFKA_SUPER_USERS="User:$ADMIN_USERNAME"
COMMAND="${@:-start}"
function start() {
./usr/bin/start-kafka.sh
}
function stop () {
kill -TERM 1
}
$COMMAND

33
kafka/templates/configmap-bin.yaml
View File

@ -1,33 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_bin }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kafka-bin
data:
kafka.sh: |
{{ tuple "bin/_kafka.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
kafka-liveness.sh: |
{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
kafka-readiness.sh: |
{{ tuple "bin/_kafka-probe.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
helm-test.sh: |
{{ tuple "bin/_helm-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
generate-acl.sh: |
{{ tuple "bin/_generate-acl.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end -}}

25
kafka/templates/configmap-etc.yaml
View File

@ -1,25 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.configmap_etc }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Secret
metadata:
name: kafka-etc
type: Opaque
data:
{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.kafka.jaas.template "key" "jaas.conf" "format" "Secret") | indent 2 }}
{{- end }}

18
kafka/templates/ingress-kafka.yaml
View File

@ -1,18 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.ingress .Values.network.kafka.ingress.public }}
{{- $ingressOpts := dict "envAll" . "backendService" "kafka" "backendServiceType" "kafka" "backendPort" "broker" -}}
{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
{{- end }}

72
kafka/templates/job-generate-acl.yaml
View File

@ -1,72 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_generate_acl }}
{{- $envAll := . }}
{{- $KafkaUserSecret := .Values.secrets.kafka.admin }}
{{- $serviceAccountName := "kafka-generate-acl" }}
{{ tuple $envAll "generate_acl" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: kafka-generate-acl
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
backoffLimit: {{ .Values.jobs.generate_acl.backoffLimit }}
template:
metadata:
labels:
{{ tuple $envAll "kafka" "generate-acl" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "generate-acl" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
activeDeadlineSeconds: {{ .Values.jobs.generate_acl.activeDeadlineSeconds }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }}
initContainers:
{{ tuple $envAll "generate_acl" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: generate-acl
{{ tuple $envAll "generate_acl" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.generate_acl | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "generate_acl" "container" "generate_acl" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: KAFKA_ZOOKEEPER_CONNECT
value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
command:
- /tmp/generate-acl.sh
volumeMounts:
- name: kafka-bin
mountPath: /tmp/generate-acl.sh
subPath: generate-acl.sh
readOnly: true
- name: kafka-etc
mountPath: /opt/kafka/config/jaas.conf
subPath: jaas.conf
readOnly: true
volumes:
- name: kafka-bin
configMap:
name: kafka-bin
defaultMode: 0555
- name: kafka-etc
secret:
secretName: kafka-etc
defaultMode: 0444
{{- end }}

18
kafka/templates/job-image-repo-sync.yaml
View File

@ -1,18 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }}
{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kafka" -}}
{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }}
{{- end }}

31
kafka/templates/monitoring/prometheus/bin/_kafka-exporter.sh.tpl
View File

@ -1,31 +0,0 @@
#!/bin/sh
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
COMMAND="${@:-start}"
function start () {
exec /bin/kafka_exporter \
--sasl.enabled \
--sasl.username=$KAFKA_EXPORTER_USERNAME \
--sasl.password=$KAFKA_EXPORTER_PASSWORD \
--kafka.server=$KAFKA_BROKERS
}
function stop () {
kill -TERM 1
}
$COMMAND

25
kafka/templates/monitoring/prometheus/configmap-bin.yaml
View File

@ -1,25 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kafka-exporter-bin
data:
kafka-exporter.sh: |
{{ tuple "bin/_kafka-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

98
kafka/templates/monitoring/prometheus/deployment.yaml
View File

@ -1,98 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.deployment .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
{{- $kafkaExporterUserSecret := .Values.secrets.kafka_exporter.user }}
{{- $serviceAccountName := "prometheus-kafka-exporter" }}
{{ tuple $envAll "kafka_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-kafka-exporter
labels:
{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
replicas: {{ .Values.pod.replicas.kafka_exporter }}
selector:
matchLabels:
{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
template:
metadata:
labels:
{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
spec:
{{ dict "envAll" $envAll "application" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }}
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka_exporter.timeout | default "30" }}
initContainers:
{{ tuple $envAll "kafka_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: kafka-exporter
{{ tuple $envAll "kafka_exporter" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.kafka_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "kafka_exporter" "container" "kafka_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
command:
- /tmp/kafka-exporter.sh
- start
lifecycle:
preStop:
exec:
command:
- /tmp/kafka-exporter.sh
- stop
env:
- name: KAFKA_BROKERS
value: {{ tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | quote }}
- name: KAFKA_EXPORTER_USERNAME
valueFrom:
secretKeyRef:
name: {{ $kafkaExporterUserSecret }}
key: KAFKA_EXPORTER_USERNAME
- name: KAFKA_EXPORTER_PASSWORD
valueFrom:
secretKeyRef:
name: {{ $kafkaExporterUserSecret }}
key: KAFKA_EXPORTER_PASSWORD
ports:
- name: exporter
containerPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
readinessProbe:
tcpSocket:
port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
initialDelaySeconds: 20
periodSeconds: 10
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: kafka-exporter-bin
mountPath: /tmp/kafka-exporter.sh
subPath: kafka-exporter.sh
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: kafka-exporter-bin
configMap:
name: kafka-exporter-bin
defaultMode: 0555
{{- end }}

18
kafka/templates/monitoring/prometheus/network-policy.yaml
View File

@ -1,18 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.network_policy .Values.monitoring.prometheus.enabled -}}
{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-kafka-exporter" -}}
{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
{{- end -}}

27
kafka/templates/monitoring/prometheus/secret-exporter.yaml
View File

@ -1,27 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.secret_kafka }}
{{- $envAll := . }}
{{- $secretName := .Values.secrets.kafka_exporter.user }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
KAFKA_EXPORTER_USERNAME: {{ .Values.endpoints.kafka_exporter.auth.username | b64enc }}
KAFKA_EXPORTER_PASSWORD: {{ .Values.endpoints.kafka_exporter.auth.password | b64enc }}
{{- end }}

36
kafka/templates/monitoring/prometheus/service.yaml
View File

@ -1,36 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.monitoring.prometheus.service .Values.monitoring.prometheus.enabled }}
{{- $envAll := . }}
{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kafka_exporter }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "kafka_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
labels:
{{ tuple $envAll "kafka-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
{{- if .Values.monitoring.prometheus.enabled }}
{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
{{- end }}
spec:
ports:
- name: exporter
port: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
targetPort: {{ tuple "kafka_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
selector:
{{ tuple $envAll "kafka-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{- end }}

17
kafka/templates/network_policy.yaml
View File

@ -1,17 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */}}
{{- if .Values.manifests.network_policy -}}
{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kafka" -}}
{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }}
{{- end -}}

74
kafka/templates/pod-helm-test.yaml
View File

@ -1,74 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.helm_test }}
{{- $envAll := . }}
{{- $serviceAccountName := print .Release.Name "-test" }}
{{ tuple $envAll "test" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: v1
kind: Pod
metadata:
name: "{{.Release.Name}}-test"
labels:
{{ tuple $envAll "kafka" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
annotations:
"helm.sh/hook": test-success
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }}
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
restartPolicy: Never
initContainers:
{{ tuple $envAll "test" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
containers:
- name: {{.Release.Name}}-helm-test
{{ tuple $envAll "helm_test" | include "helm-toolkit.snippets.image" | indent 6 }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.test | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
{{ dict "envAll" $envAll "application" "test" "container" "helm_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }}
command:
- "/tmp/helm-test.sh"
env:
- name: KAFKA_ZOOKEEPER_CONNECT
value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
- name: KAFKA_BROKERS
value: "{{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
- name: KAFKA_OPTS
value: {{ include "helm-toolkit.utils.joinListWithSpace" .Values.conf.kafka.jvm_options | quote }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: kafka-bin
mountPath: /tmp/helm-test.sh
subPath: helm-test.sh
readOnly: true
- name: kafka-etc
mountPath: /opt/kafka/config/jaas.conf
subPath: jaas.conf
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: kafka-bin
configMap:
name: kafka-bin
defaultMode: 0555
- name: kafka-etc
secret:
secretName: kafka-etc
defaultMode: 0444
{{- end }}

17
kafka/templates/secret-ingress-tls.yaml
View File

@ -1,17 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.secret_ingress_tls -}}
{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "kafka" "backendService" "kafka" ) }}
{{- end }}

27
kafka/templates/secret-kafka.yaml
View File

@ -1,27 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.secret_kafka }}
{{- $envAll := . }}
{{- $secretName := .Values.secrets.kafka.admin }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
type: Opaque
data:
KAFKA_ADMIN_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }}
KAFKA_ADMIN_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }}
{{- end }}

32
kafka/templates/service-discovery.yaml
View File

@ -1,32 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service_discovery }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "kafka" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
labels:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
ports:
- name: broker
targetPort: broker
port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
clusterIP: None
selector:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{- end }}

18
kafka/templates/service-ingress-kafka.yaml
View File

@ -1,18 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and .Values.manifests.service_ingress .Values.network.kafka.ingress.public }}
{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kafka" -}}
{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
{{- end }}

36
kafka/templates/service.yaml
View File

@ -1,36 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.service }}
{{- $envAll := . }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ tuple "kafka" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
labels:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
ports:
- name: broker
port: {{ tuple "kafka" "internal" "broker" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{ if .Values.network.kafka.node_port.enabled }}
nodePort: {{ .Values.network.kafka.node_port.port }}
{{ end }}
selector:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
{{ if .Values.network.kafka.node_port.enabled }}
type: NodePort
{{ end }}
{{- end }}

191
kafka/templates/statefulset.yaml
View File

@ -1,191 +0,0 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.statefulset }}
{{- $envAll := . }}
{{- $mounts_kafka := .Values.pod.mounts.kafka.kafka }}
{{- $mounts_kafka_init := .Values.pod.mounts.kafka.init_container }}
{{- $kafkaUserSecret := .Values.secrets.kafka.admin }}
{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $serviceAccountName := printf "%s-%s" .Release.Name "kafka" }}
{{ tuple $envAll "kafka" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: {{ $serviceAccountName }}
rules:
- apiGroups:
- ""
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- nonResourceURLs:
- "/metrics"
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ $serviceAccountName }}
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ $serviceAccountName }}
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: kafka
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
serviceName: {{ tuple "kafka" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
replicas: {{ .Values.pod.replicas.kafka }}
updateStrategy:
type: OnDelete
podManagementPolicy: Parallel
selector:
matchLabels:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
template:
metadata:
labels:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }}
configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }}
spec:
{{ dict "envAll" $envAll "application" "kafka" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
serviceAccountName: {{ $serviceAccountName }}
affinity:
{{ tuple $envAll "kafka" "broker" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
nodeSelector:
{{ .Values.labels.kafka.node_selector_key }}: {{ .Values.labels.kafka.node_selector_value | quote }}
terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kafka.timeout | default "30" }}
initContainers:
{{ tuple $envAll "kafka" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: kafka
command:
- "/tmp/kafka.sh"
{{ tuple $envAll "kafka" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.kafka | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "kafka" "container" "kafka" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
ports:
- name: broker
containerPort: {{ $kafkaBrokerPort }}
env:
- name: ADMIN_USERNAME
valueFrom:
secretKeyRef:
name: {{ $kafkaUserSecret }}
key: KAFKA_ADMIN_USERNAME
- name: KAFKA_PORT
value: "{{ $kafkaBrokerPort }}"
- name: ZOOKEEPER_PORT
value: "{{ tuple "zookeeper" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
- name: KAFKA_ZOOKEEPER_CONNECT
value: "{{ tuple "zookeeper" "internal" "client" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}"
- name: KAFKA_LISTENERS
value: "PLAINTEXT://:{{$kafkaBrokerPort}}"
- name: KAFKA_CREATE_TOPICS
value: "{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.kafka.topics }}"
- name: KAFKA_OPTS
value: {{ include "helm-toolkit.utils.joinListWithSpace" .Values.conf.kafka.jvm_options | quote }}
readinessProbe:
initialDelaySeconds: 20
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 2
successThreshold: 1
exec:
command:
- /tmp/kafka-readiness.sh
livenessProbe:
initialDelaySeconds: 20
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 2
successThreshold: 1
exec:
command:
- /tmp/kafka-liveness.sh
volumeMounts:
- name: kafka-bin
mountPath: /tmp/kafka.sh
subPath: kafka.sh
readOnly: true
- name: kafka-bin
mountPath: /tmp/kafka-liveness.sh
subPath: kafka-liveness.sh
readOnly: true
- name: kafka-bin
mountPath: /tmp/kafka-readiness.sh
subPath: kafka-readiness.sh
readOnly: true
- name: kafka-etc
mountPath: /opt/kafka/config/jaas.conf
subPath: jaas.conf
readOnly: true
- name: data
mountPath: {{ .Values.conf.kafka.config.data_directory }}
{{ if $mounts_kafka.volumeMounts }}{{ toYaml $mounts_kafka.volumeMounts | indent 12 }}{{ end }}
volumes:
- name: kafka-bin
configMap:
name: kafka-bin
defaultMode: 0555
- name: kafka-etc
secret:
secretName: kafka-etc
defaultMode: 0444
{{ if $mounts_kafka.volumes }}{{ toYaml $mounts_kafka.volumes | indent 8 }}{{ end }}
{{- if not .Values.storage.enabled }}
- name: data
emptyDir: {}
{{- else }}
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: {{ .Values.storage.pvc.access_mode }}
resources:
requests:
storage: {{ .Values.storage.requests.storage }}
storageClassName: {{ .Values.storage.storage_class }}
{{- end }}
{{- end }}

379
kafka/values.yaml
View File

@ -1,379 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Default values for kafka.
# This is a YAML-formatted file.
# Declare name/value pairs to be passed into your templates.
# name: value
---
images:
tags:
kafka: docker.io/wurstmeister/kafka:2.12-2.3.0
kafka_exporter: docker.io/danielqsj/kafka-exporter:latest
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
image_repo_sync: docker.io/docker:17.07.0
helm_test: docker.io/wurstmeister/kafka:2.12-2.3.0
generate_acl: docker.io/wurstmeister/kafka:2.12-2.3.0
pull_policy: IfNotPresent
local_registry:
active: false
exclude:
- dep_check
- image_repo_sync
labels:
kafka:
node_selector_key: openstack-control-plane
node_selector_value: enabled
job:
node_selector_key: openstack-control-plane
node_selector_value: enabled
test:
node_selector_key: openstack-control-plane
node_selector_value: enabled
pod:
security_context:
kafka:
pod: {}
container:
kafka: {}
kafka-init: {}
kafka_exporter:
pod: {}
container:
kafka_exporter: {}
generate_acl:
pod: {}
container:
generate_acl: {}
affinity:
anti:
type:
default: preferredDuringSchedulingIgnoredDuringExecution
topologyKey:
default: kubernetes.io/hostname
weight:
default: 10
mounts:
kafka:
kafka:
init_container: null
replicas:
kafka: 3
kafka_exporter: 1
lifecycle:
upgrades:
statefulsets:
pod_replacement_strategy: RollingUpdate
termination_grace_period:
kafka:
timeout: 30
kafka_exporter:
timeout: 30
resources:
enabled: false
kafka:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "128Mi"
cpu: "500m"
kafka_exporter:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
jobs:
image_repo_sync:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
generate_acl:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
test:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "1024Mi"
cpu: "2000m"
endpoints:
cluster_domain_suffix: cluster.local
local_image_registry:
name: docker-registry
namespace: docker-registry
hosts:
default: localhost
internal: docker-registry
node: localhost
host_fqdn_override:
default: null
port:
registry:
node: 5000
kafka:
name: kafka
namespace: null
auth:
admin:
username: admin
password: changeme
hosts:
default: kafka-broker
discovery: kafka-discovery
public: kafka
host_fqdn_override:
default: null
# NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public
# endpoints using the following format:
# public:
# host: null
# tls:
# crt: null
# key: null
path:
default: null
scheme:
default: 'http'
port:
broker:
default: 9092
kafka-exporter:
default: 9141
jmx-exporter:
default: 9404
kafka_exporter:
auth:
username: kafka-exporter
password: changeme
namespace: null
hosts:
default: kafka-exporter
host_fqdn_override:
default: null
scheme:
default: 'http'
port:
exporter:
default: 9308
zookeeper:
name: zookeeper
namespace: null
auth:
admin:
username: admin
password: changeme
hosts:
default: zookeeper-int
public: zookeeper
host_fqdn_override:
default: null
path:
default: null
scheme:
default: 'http'
port:
client:
default: 2181
server:
default: 2888
dependencies:
dynamic:
common:
local_image_registry:
jobs:
- kafka-image-repo-sync
services:
- endpoint: node
service: local_image_registry
static:
image_repo_sync:
services:
- endpoint: internal
service: local_image_registry
kafka:
services:
- endpoint: internal
service: zookeeper
kafka_exporter:
services:
- endpoint: internal
service: kafka
generate_acl:
services:
- endpoint: internal
service: kafka
monitoring:
prometheus:
enabled: true
kafka_exporter:
scrape: true
network:
kafka:
ingress:
public: true
classes:
namespace: "nginx"
cluster: "nginx-cluster"
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-kafka
nginx.ingress.kubernetes.io/session-cookie-hash: sha1
nginx.ingress.kubernetes.io/session-cookie-expires: "600"
nginx.ingress.kubernetes.io/session-cookie-max-age: "600"
node_port:
enabled: false
port: 31033
network_policy:
kafka:
ingress:
- {}
egress:
- {}
kafka_exporter:
ingress:
- {}
egress:
- {}
secrets:
tls:
kafka:
kafka:
public: kafka-tls-public
kafka:
admin: kafka-admin-creds
kafka_exporter:
user: kafka-exporter-creds
storage:
enabled: true
pvc:
name: kafka-pvc
access_mode: ["ReadWriteOnce"]
requests:
storage: 5Gi
storage_class: general
manifests:
configmap_bin: true
configmap_etc: true
helm_test: true
ingress: true
job_image_repo_sync: true
job_generate_acl: true
monitoring:
prometheus:
configmap_bin: true
deployment: true
secret_exporter: true
service: true
network_policy: false
network_policy: false
secret_ingress_tls: true
secret_kafka: true
secret_zookeeper: true
service_discovery: true
service_ingress: true
service: true
statefulset: true
jobs:
generate_acl:
backoffLimit: 6
activeDeadlineSeconds: 600
conf:
kafka:
config:
data_directory: /var/lib/kafka/data
server_settings:
# Optionally provide configuration overrides for Kafka's
# server.properties file. Replace '.' with '_' ie:
# for message.max.bytes enter message_max_bytes
message_max_bytes: 5000000
authorizer_class_name: kafka.security.auth.SimpleAclAuthorizer
listeners: SASL_PLAINTEXT://:9092
security_protocol: SASL_PLAINTEXT
security_inter_broker_protocol: SASL_PLAINTEXT
sasl_mechanism: PLAIN
sasl_enabled_mechanisms: PLAIN
sasl_mechanism_inter_broker_protocol: PLAIN
topics:
# List of topic strings formatted like:
# topic_name:number_of_partitions:replication_factor
# - "mytopic:1:1"
jaas: # Define Authentication Details in this section
producers:
# region_a: # Just an ID used to iterate through the dict of producers
# username: region-a-producer
# password: changeme
# topic: region-a # Used in generate-acl.sh to provide access
consumers:
# region_a: # Just an ID used to iterate through the dict of consumers
# username: region-a-consumer
# password: changeme
# topic: region-a # Used in generate-acl.sh to provide access
# group: region-a # Used in generate-acl.sh to provide access
template: |
KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
{{- $admin := .Values.endpoints.kafka.auth.admin }}
username={{ $admin.username | quote}}
password={{ $admin.password | quote}}
user_{{ $admin.username }}={{ $admin.password | quote }}
{{- if .Values.monitoring.prometheus.enabled }}
{{- $exporter := .Values.endpoints.kafka_exporter.auth }}
user_{{ $exporter.username }}={{ $exporter.password | quote }}
{{- end }}
{{- range $producer, $credentials := .Values.conf.kafka.jaas.producers }}
user_{{ $credentials.username }}={{ $credentials.password | quote }}
{{- end }}
{{- range $consumer, $credentials := .Values.conf.kafka.jaas.producers }}
user_{{ $credentials.username }}={{ $credentials.password | quote }}
{{- end }}
{{- printf ";" }}
};
KafkaClient {
org.apache.kafka.common.security.plain.PlainLoginModule required
username={{ $admin.username | quote}}
password={{ $admin.password | quote}}
{{- printf ";" }}
};
Client {
org.apache.kafka.common.security.plain.PlainLoginModule required
username={{ $admin.username | quote}}
password={{ $admin.password | quote}}
{{- printf ";" }}
};
jvm_options:
- -Djava.security.auth.login.config=/opt/kafka/config/jaas.conf
...

2
releasenotes/config.yaml
View File

@ -24,7 +24,6 @@ sections:
- [grafana, grafana Chart] - [grafana, grafana Chart]
- [helm-toolkit, helm-toolkit Chart] - [helm-toolkit, helm-toolkit Chart]
- [ingress, ingress Chart] - [ingress, ingress Chart]
- [kafka, kafka Chart]
- [kibana, kibana Chart] - [kibana, kibana Chart]
- [kube-dns, kube-dns Chart] - [kube-dns, kube-dns Chart]
- [kubernetes-keystone-webhook, kubernetes-keystone-webhook Chart] - [kubernetes-keystone-webhook, kubernetes-keystone-webhook Chart]
@ -55,7 +54,6 @@ sections:
- [redis, redis Chart] - [redis, redis Chart]
- [registry, registry Chart] - [registry, registry Chart]
- [tiller, tiller Chart] - [tiller, tiller Chart]
- [zookeeper, zookeeper Chart]
- [features, New Features] - [features, New Features]
- [issues, Known Issues] - [issues, Known Issues]
- [upgrade, Upgrade Notes] - [upgrade, Upgrade Notes]

5
releasenotes/notes/kafka.yaml
View File

@ -1,5 +0,0 @@
---
kafka:
- 0.1.0 Initial Chart
- 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
...

5
releasenotes/notes/zookeeper.yaml
View File

@ -1,5 +0,0 @@
---
zookeeper:
- 0.1.0 Initial Chart
- 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0"
...

1
tools/deployment/osh-infra-kafka/000-install-packages.sh
View File

@ -1 +0,0 @@
../common/000-install-packages.sh

1
tools/deployment/osh-infra-kafka/005-deploy-k8s.sh
View File

@ -1 +0,0 @@
../common/005-deploy-k8s.sh

1
tools/deployment/osh-infra-kafka/010-ingress.sh
View File

@ -1 +0,0 @@
../osh-infra-logging/010-ingress.sh

1
tools/deployment/osh-infra-kafka/020-ceph.sh
View File

@ -1 +0,0 @@
../osh-infra-logging/020-ceph.sh

1
tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh
View File

@ -1 +0,0 @@
../osh-infra-logging/025-ceph-ns-activate.sh

1
tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh
View File

@ -1 +0,0 @@
../osh-infra-logging/030-radosgw-osh-infra.sh

1
tools/deployment/osh-infra-kafka/040-zookeeper.sh
View File

@ -1 +0,0 @@
../common/zookeeper.sh

33
tools/deployment/osh-infra-kafka/050-kafka.sh
View File

@ -1,33 +0,0 @@
#!/bin/bash
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Lint and package chart
make kafka
#NOTE: Deploy command
helm upgrade --install kafka ./kafka \
--namespace=osh-infra \
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh osh-infra
#NOTE: Validate deployment info
helm status kafka
# Delete the test pod if it still exists
kubectl delete pods -l application=kafka,release_group=kafka,component=test --namespace=osh-infra --ignore-not-found
#NOTE: Test deployment
helm test kafka

20
zuul.d/jobs.yaml
View File

@ -158,26 +158,6 @@
- ./tools/deployment/osh-infra-logging/070-kibana.sh - ./tools/deployment/osh-infra-logging/070-kibana.sh
- ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true - ./tools/deployment/osh-infra-logging/600-kibana-selenium.sh || true
- job:
name: openstack-helm-infra-kafka
parent: openstack-helm-infra-functional
timeout: 7200
pre-run:
- playbooks/osh-infra-upgrade-host.yaml
post-run: playbooks/osh-infra-collect-logs.yaml
nodeset: openstack-helm-single-node
vars:
gate_scripts_relative_path: ../openstack-helm-infra
gate_scripts:
- ./tools/deployment/osh-infra-kafka/000-install-packages.sh
- ./tools/deployment/osh-infra-kafka/005-deploy-k8s.sh
- ./tools/deployment/osh-infra-kafka/010-ingress.sh
- ./tools/deployment/osh-infra-kafka/020-ceph.sh
- ./tools/deployment/osh-infra-kafka/025-ceph-ns-activate.sh
- ./tools/deployment/osh-infra-kafka/030-radosgw-osh-infra.sh
- ./tools/deployment/osh-infra-kafka/040-zookeeper.sh
- ./tools/deployment/osh-infra-kafka/050-kafka.sh
- job: - job:
name: openstack-helm-infra-aio-monitoring name: openstack-helm-infra-aio-monitoring
parent: openstack-helm-infra-functional parent: openstack-helm-infra-functional

1
zuul.d/project.yaml
View File

@ -67,7 +67,6 @@
# - openstack-helm-infra-armada-update-uuid # - openstack-helm-infra-armada-update-uuid
# - openstack-helm-infra-armada-update-passwords # - openstack-helm-infra-armada-update-passwords
- openstack-helm-infra-federated-monitoring - openstack-helm-infra-federated-monitoring
- openstack-helm-infra-kafka
- openstack-helm-infra-local-storage - openstack-helm-infra-local-storage
- openstack-helm-infra-aio-network-policy - openstack-helm-infra-aio-network-policy
- openstack-helm-infra-apparmor - openstack-helm-infra-apparmor