c74b147fe7
These are mostly CI fixes: * Use podman+cri-o based minikube: * This is still considered experimental, but seems to be more supported than the 'none' driver. * Fix an issue where ssh to the emulated static node fails: * PAM needed to be disabled for openssh * openssh needs more permissions to run - cri-o based minikube is more strict * Rebase test container to Fedora 40 * Update the ingress definition to current API version * Update zookeeper from 3.5.5 to 3.8.4: * required for nodepool 9.0.0+ * Update the percona operator from 1.11 to 1.14: * required for kubernetes 1.24+ * Update test node to Ubuntu Jammy from Ubuntu Bionic * Update minikube to 1.33.1 * Added some more explicit logging to the k8s state, this could be split off into a role in future. Depends-On: https://review.opendev.org/c/zuul/zuul-jobs/+/924970 Change-Id: I7bf27750073fa807069af6f85f2689173b278abe
369 lines
11 KiB
YAML
369 lines
11 KiB
YAML
---
|
|
apiVersion: cert-manager.io/v1
|
|
kind: Certificate
|
|
metadata:
|
|
name: zookeeper-server
|
|
spec:
|
|
privateKey:
|
|
encoding: PKCS8
|
|
secretName: zookeeper-server-tls
|
|
commonName: server
|
|
usages:
|
|
- digital signature
|
|
- key encipherment
|
|
- server auth
|
|
- client auth
|
|
dnsNames:
|
|
- zookeeper-0.zookeeper-headless.{{ namespace }}.svc.cluster.local
|
|
- zookeeper-0
|
|
- zookeeper-1.zookeeper-headless.{{ namespace }}.svc.cluster.local
|
|
- zookeeper-1
|
|
- zookeeper-2.zookeeper-headless.{{ namespace }}.svc.cluster.local
|
|
- zookeeper-2
|
|
issuerRef:
|
|
name: ca-issuer
|
|
kind: Issuer
|
|
---
|
|
# Source: zookeeper/templates/poddisruptionbudget.yaml
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata:
|
|
name: zookeeper
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
maxUnavailable: 1
|
|
---
|
|
# Source: zookeeper/templates/config-script.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: zookeeper
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
data:
|
|
ok: |
|
|
#!/bin/sh
|
|
if [ -f /tls/client/ca.crt ]; then
|
|
echo "srvr" | openssl s_client -CAfile /tls/client/ca.crt -cert /tls/client/tls.crt -key /tls/client/tls.key -connect 127.0.0.1:${1:-2281} -quiet -ign_eof 2>/dev/null | grep Mode
|
|
else
|
|
zkServer.sh status
|
|
fi
|
|
|
|
ready: |
|
|
#!/bin/sh
|
|
if [ -f /tls/client/ca.crt ]; then
|
|
echo "ruok" | openssl s_client -CAfile /tls/client/ca.crt -cert /tls/client/tls.crt -key /tls/client/tls.key -connect 127.0.0.1:${1:-2281} -quiet -ign_eof 2>/dev/null
|
|
else
|
|
echo ruok | nc 127.0.0.1 ${1:-2181}
|
|
fi
|
|
|
|
run: |
|
|
#!/bin/bash
|
|
|
|
set -a
|
|
ROOT=$(echo /apache-zookeeper-*)
|
|
|
|
ZK_USER=${ZK_USER:-"zookeeper"}
|
|
ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"}
|
|
ZK_DATA_DIR=${ZK_DATA_DIR:-"/data"}
|
|
ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/data/log"}
|
|
ZK_CONF_DIR=${ZK_CONF_DIR:-"/conf"}
|
|
ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
|
|
ZK_SSL_CLIENT_PORT=${ZK_SSL_CLIENT_PORT:-2281}
|
|
ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
|
|
ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
|
|
ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
|
|
ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
|
|
ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
|
|
ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G}
|
|
ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60}
|
|
ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))}
|
|
ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))}
|
|
ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3}
|
|
ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0}
|
|
ID_FILE="$ZK_DATA_DIR/myid"
|
|
ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg"
|
|
LOG4J_PROPERTIES="$ZK_CONF_DIR/log4j.properties"
|
|
HOST=$(hostname)
|
|
DOMAIN=`hostname -d`
|
|
JVMFLAGS="-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE"
|
|
|
|
APPJAR=$(echo $ROOT/*jar)
|
|
CLASSPATH="${ROOT}/lib/*:${APPJAR}:${ZK_CONF_DIR}:"
|
|
|
|
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
|
|
NAME=${BASH_REMATCH[1]}
|
|
ORD=${BASH_REMATCH[2]}
|
|
MY_ID=$((ORD+1))
|
|
else
|
|
echo "Failed to extract ordinal from hostname $HOST"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p $ZK_DATA_DIR
|
|
mkdir -p $ZK_DATA_LOG_DIR
|
|
echo $MY_ID >> $ID_FILE
|
|
|
|
if [[ -f /tls/server/ca.crt ]]; then
|
|
cp /tls/server/ca.crt /data/server-ca.pem
|
|
cat /tls/server/tls.crt /tls/server/tls.key > /data/server.pem
|
|
fi
|
|
if [[ -f /tls/client/ca.crt ]]; then
|
|
cp /tls/client/ca.crt /data/client-ca.pem
|
|
cat /tls/client/tls.crt /tls/client/tls.key > /data/client.pem
|
|
fi
|
|
|
|
echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
|
|
echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
|
|
echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
|
|
echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
|
|
echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
|
|
echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
|
|
echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
|
|
echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
|
|
echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
|
|
echo "autopurge.purgeInterval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
|
|
echo "4lw.commands.whitelist=*" >> $ZK_CONFIG_FILE
|
|
|
|
# Client TLS configuration
|
|
if [[ -f /tls/client/ca.crt ]]; then
|
|
echo "secureClientPort=$ZK_SSL_CLIENT_PORT" >> $ZK_CONFIG_FILE
|
|
echo "ssl.keyStore.location=/data/client.pem" >> $ZK_CONFIG_FILE
|
|
echo "ssl.trustStore.location=/data/client-ca.pem" >> $ZK_CONFIG_FILE
|
|
else
|
|
echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
|
|
fi
|
|
|
|
# Server TLS configuration
|
|
if [[ -f /tls/server/ca.crt ]]; then
|
|
echo "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory" >> $ZK_CONFIG_FILE
|
|
echo "sslQuorum=true" >> $ZK_CONFIG_FILE
|
|
echo "ssl.quorum.keyStore.location=/data/server.pem" >> $ZK_CONFIG_FILE
|
|
echo "ssl.quorum.trustStore.location=/data/server-ca.pem" >> $ZK_CONFIG_FILE
|
|
fi
|
|
|
|
for (( i=1; i<=$ZK_REPLICAS; i++ ))
|
|
do
|
|
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT" >> $ZK_CONFIG_FILE
|
|
done
|
|
|
|
rm -f $LOG4J_PROPERTIES
|
|
|
|
echo "zookeeper.root.logger=$ZK_LOG_LEVEL, CONSOLE" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.console.threshold=$ZK_LOG_LEVEL" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.log.threshold=$ZK_LOG_LEVEL" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.log.dir=$ZK_DATA_LOG_DIR" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.log.file=zookeeper.log" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.log.maxfilesize=256MB" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.log.maxbackupindex=10" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.tracelog.dir=$ZK_DATA_LOG_DIR" >> $LOG4J_PROPERTIES
|
|
echo "zookeeper.tracelog.file=zookeeper_trace.log" >> $LOG4J_PROPERTIES
|
|
echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOG4J_PROPERTIES
|
|
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOG4J_PROPERTIES
|
|
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOG4J_PROPERTIES
|
|
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOG4J_PROPERTIES
|
|
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOG4J_PROPERTIES
|
|
|
|
if [ -n "$JMXDISABLE" ]
|
|
then
|
|
MAIN=org.apache.zookeeper.server.quorum.QuorumPeerMain
|
|
else
|
|
MAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=$JMXPORT -Dcom.sun.management.jmxremote.authenticate=$JMXAUTH -Dcom.sun.management.jmxremote.ssl=$JMXSSL -Dzookeeper.jmx.log4j.disable=$JMXLOG4J org.apache.zookeeper.server.quorum.QuorumPeerMain"
|
|
fi
|
|
|
|
set -x
|
|
exec java -cp "$CLASSPATH" $JVMFLAGS $MAIN $ZK_CONFIG_FILE
|
|
---
|
|
# Source: zookeeper/templates/service-headless.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: zookeeper-headless
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
spec:
|
|
clusterIP: None
|
|
publishNotReadyAddresses: true
|
|
ports:
|
|
- name: client
|
|
port: 2281
|
|
targetPort: client
|
|
protocol: TCP
|
|
- name: election
|
|
port: 3888
|
|
targetPort: election
|
|
protocol: TCP
|
|
- name: server
|
|
port: 2888
|
|
targetPort: server
|
|
protocol: TCP
|
|
selector:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
---
|
|
# Source: zookeeper/templates/service.yaml
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: zookeeper
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
spec:
|
|
type: ClusterIP
|
|
ports:
|
|
- name: client
|
|
port: 2281
|
|
protocol: TCP
|
|
targetPort: client
|
|
selector:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
---
|
|
# Source: zookeeper/templates/statefulset.yaml
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: zookeeper
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
spec:
|
|
serviceName: zookeeper-headless
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
podManagementPolicy: Parallel
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: zookeeper
|
|
release: zookeeper
|
|
component: server
|
|
spec:
|
|
terminationGracePeriodSeconds: 1800
|
|
securityContext:
|
|
fsGroup: 1000
|
|
runAsUser: 1000
|
|
containers:
|
|
|
|
- name: zookeeper
|
|
image: "docker.io/library/zookeeper:3.8.4"
|
|
imagePullPolicy: IfNotPresent
|
|
command:
|
|
- "/bin/bash"
|
|
- "-xec"
|
|
- "/config-scripts/run"
|
|
ports:
|
|
- name: client
|
|
containerPort: 2281
|
|
protocol: TCP
|
|
- name: election
|
|
containerPort: 3888
|
|
protocol: TCP
|
|
- name: server
|
|
containerPort: 2888
|
|
protocol: TCP
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- sh
|
|
- /config-scripts/ok
|
|
initialDelaySeconds: 20
|
|
periodSeconds: 30
|
|
timeoutSeconds: 5
|
|
failureThreshold: 2
|
|
successThreshold: 1
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- sh
|
|
- /config-scripts/ready
|
|
initialDelaySeconds: 20
|
|
periodSeconds: 30
|
|
timeoutSeconds: 5
|
|
failureThreshold: 2
|
|
successThreshold: 1
|
|
env:
|
|
- name: ZK_REPLICAS
|
|
value: "3"
|
|
- name: JMXAUTH
|
|
value: "false"
|
|
- name: JMXDISABLE
|
|
value: "false"
|
|
- name: JMXPORT
|
|
value: "1099"
|
|
- name: JMXSSL
|
|
value: "false"
|
|
- name: ZK_SYNC_LIMIT
|
|
value: "10"
|
|
- name: ZK_TICK_TIME
|
|
value: "2000"
|
|
- name: ZOO_AUTOPURGE_PURGEINTERVAL
|
|
value: "0"
|
|
- name: ZOO_AUTOPURGE_SNAPRETAINCOUNT
|
|
value: "3"
|
|
- name: ZOO_INIT_LIMIT
|
|
value: "5"
|
|
- name: ZOO_MAX_CLIENT_CNXNS
|
|
value: "60"
|
|
- name: ZOO_PORT
|
|
value: "2181"
|
|
- name: ZOO_STANDALONE_ENABLED
|
|
value: "false"
|
|
- name: ZOO_TICK_TIME
|
|
value: "2000"
|
|
resources:
|
|
{}
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
- name: zookeeper-server-tls
|
|
mountPath: /tls/server
|
|
readOnly: true
|
|
- name: zookeeper-client-tls
|
|
mountPath: /tls/client
|
|
readOnly: true
|
|
- name: config
|
|
mountPath: /config-scripts
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: zookeeper
|
|
defaultMode: 0555
|
|
- name: zookeeper-server-tls
|
|
secret:
|
|
secretName: zookeeper-server-tls
|
|
- name: zookeeper-client-tls
|
|
secret:
|
|
secretName: zookeeper-server-tls
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: data
|
|
spec:
|
|
accessModes:
|
|
- "ReadWriteOnce"
|
|
resources:
|
|
requests:
|
|
storage: "5Gi"
|
|
{%- if spec.storageClassName != "" %}
|
|
storageClassName: {{ spec.zookeeper.storageClassName }}
|
|
{%- endif %}
|