zuul/zuul-operator
Code Issues Proposed changes
d65ebc6bff
zuul-operator/zuul_operator/templates/zookeeper.yaml
James E. Blair eff9f360f7 Use kopf operator framework 
This switches from the ansible/dhall operator framework to kopf,
an operator framework written in pure Python.  This allows us to:

* Build the operator application as a Python app.
* Build the operator image using the opendev python builder images.
* Run the operator as a Python CLI program "zuul-operator".
* Write procedural Python code to handle operator tasks (such as
  creating new nodepool launchers when providers are added).
* Use Jinja for templating config files and k8s resource files
  (direct pythonic manipulation of resources is an option too).

The new CR nearly matches the existing one, with some minor differences.

Some missing features and documentation are added in the commits
immediately following; they should be reviewed and merged as a unit.

Also, fx waiting for scheduler to settle in functional test since
we changed this log line in Zuul.

Change-Id: Ib37b67e3444b7cd44692d48eee77775ee9049e9f

Change-Id: I70ec31ecd8fe264118215944022b2e7b513dced9
2021-07-20 13:16:07 -07:00

365 lines
11 KiB
YAML
Raw Blame History

---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
name: zookeeper-server
spec:
keyEncoding: pkcs8
secretName: zookeeper-server-tls
commonName: server
usages:
- digital signature
- key encipherment
- server auth
- client auth
dnsNames:
- zookeeper-0.zookeeper-headless.{{ namespace }}.svc.cluster.local
- zookeeper-0
- zookeeper-1.zookeeper-headless.{{ namespace }}.svc.cluster.local
- zookeeper-1
- zookeeper-2.zookeeper-headless.{{ namespace }}.svc.cluster.local
- zookeeper-2
issuerRef:
name: ca-issuer
kind: Issuer
---
# Source: zookeeper/templates/poddisruptionbudget.yaml
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: zookeeper
labels:
app: zookeeper
release: zookeeper
component: server
spec:
selector:
matchLabels:
app: zookeeper
release: zookeeper
component: server
maxUnavailable: 1
---
# Source: zookeeper/templates/config-script.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: zookeeper
labels:
app: zookeeper
release: zookeeper
component: server
data:
ok: |
#!/bin/sh
if [ -f /tls/client/ca.crt ]; then
echo "srvr" | openssl s_client -CAfile /tls/client/ca.crt -cert /tls/client/tls.crt -key /tls/client/tls.key -connect 127.0.0.1:${1:-2281} -quiet -ign_eof 2>/dev/null | grep Mode
else
zkServer.sh status
fi
ready: |
#!/bin/sh
if [ -f /tls/client/ca.crt ]; then
echo "ruok" | openssl s_client -CAfile /tls/client/ca.crt -cert /tls/client/tls.crt -key /tls/client/tls.key -connect 127.0.0.1:${1:-2281} -quiet -ign_eof 2>/dev/null
else
echo ruok | nc 127.0.0.1 ${1:-2181}
fi
run: |
#!/bin/bash
set -a
ROOT=$(echo /apache-zookeeper-*)
ZK_USER=${ZK_USER:-"zookeeper"}
ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"}
ZK_DATA_DIR=${ZK_DATA_DIR:-"/data"}
ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/data/log"}
ZK_CONF_DIR=${ZK_CONF_DIR:-"/conf"}
ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
ZK_SSL_CLIENT_PORT=${ZK_SSL_CLIENT_PORT:-2281}
ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G}
ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-60}
ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))}
ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))}
ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3}
ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0}
ID_FILE="$ZK_DATA_DIR/myid"
ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg"
LOG4J_PROPERTIES="$ZK_CONF_DIR/log4j.properties"
HOST=$(hostname)
DOMAIN=`hostname -d`
JVMFLAGS="-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE"
APPJAR=$(echo $ROOT/*jar)
CLASSPATH="${ROOT}/lib/*:${APPJAR}:${ZK_CONF_DIR}:"
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
NAME=${BASH_REMATCH[1]}
ORD=${BASH_REMATCH[2]}
MY_ID=$((ORD+1))
else
echo "Failed to extract ordinal from hostname $HOST"
exit 1
fi
mkdir -p $ZK_DATA_DIR
mkdir -p $ZK_DATA_LOG_DIR
echo $MY_ID >> $ID_FILE
if [[ -f /tls/server/ca.crt ]]; then
cp /tls/server/ca.crt /data/server-ca.pem
cat /tls/server/tls.crt /tls/server/tls.key > /data/server.pem
fi
if [[ -f /tls/client/ca.crt ]]; then
cp /tls/client/ca.crt /data/client-ca.pem
cat /tls/client/tls.crt /tls/client/tls.key > /data/client.pem
fi
echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
echo "autopurge.purgeInterval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
echo "4lw.commands.whitelist=*" >> $ZK_CONFIG_FILE
# Client TLS configuration
if [[ -f /tls/client/ca.crt ]]; then
echo "secureClientPort=$ZK_SSL_CLIENT_PORT" >> $ZK_CONFIG_FILE
echo "ssl.keyStore.location=/data/client.pem" >> $ZK_CONFIG_FILE
echo "ssl.trustStore.location=/data/client-ca.pem" >> $ZK_CONFIG_FILE
else
echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
fi
# Server TLS configuration
if [[ -f /tls/server/ca.crt ]]; then
echo "serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory" >> $ZK_CONFIG_FILE
echo "sslQuorum=true" >> $ZK_CONFIG_FILE
echo "ssl.quorum.keyStore.location=/data/server.pem" >> $ZK_CONFIG_FILE
echo "ssl.quorum.trustStore.location=/data/server-ca.pem" >> $ZK_CONFIG_FILE
fi
for (( i=1; i<=$ZK_REPLICAS; i++ ))
do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT" >> $ZK_CONFIG_FILE
done
rm -f $LOG4J_PROPERTIES
echo "zookeeper.root.logger=$ZK_LOG_LEVEL, CONSOLE" >> $LOG4J_PROPERTIES
echo "zookeeper.console.threshold=$ZK_LOG_LEVEL" >> $LOG4J_PROPERTIES
echo "zookeeper.log.threshold=$ZK_LOG_LEVEL" >> $LOG4J_PROPERTIES
echo "zookeeper.log.dir=$ZK_DATA_LOG_DIR" >> $LOG4J_PROPERTIES
echo "zookeeper.log.file=zookeeper.log" >> $LOG4J_PROPERTIES
echo "zookeeper.log.maxfilesize=256MB" >> $LOG4J_PROPERTIES
echo "zookeeper.log.maxbackupindex=10" >> $LOG4J_PROPERTIES
echo "zookeeper.tracelog.dir=$ZK_DATA_LOG_DIR" >> $LOG4J_PROPERTIES
echo "zookeeper.tracelog.file=zookeeper_trace.log" >> $LOG4J_PROPERTIES
echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOG4J_PROPERTIES
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOG4J_PROPERTIES
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOG4J_PROPERTIES
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOG4J_PROPERTIES
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOG4J_PROPERTIES
if [ -n "$JMXDISABLE" ]
then
MAIN=org.apache.zookeeper.server.quorum.QuorumPeerMain
else
MAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=$JMXPORT -Dcom.sun.management.jmxremote.authenticate=$JMXAUTH -Dcom.sun.management.jmxremote.ssl=$JMXSSL -Dzookeeper.jmx.log4j.disable=$JMXLOG4J org.apache.zookeeper.server.quorum.QuorumPeerMain"
fi
set -x
exec java -cp "$CLASSPATH" $JVMFLAGS $MAIN $ZK_CONFIG_FILE
---
# Source: zookeeper/templates/service-headless.yaml
apiVersion: v1
kind: Service
metadata:
name: zookeeper-headless
labels:
app: zookeeper
release: zookeeper
spec:
clusterIP: None
publishNotReadyAddresses: true
ports:
- name: client
port: 2281
targetPort: client
protocol: TCP
- name: election
port: 3888
targetPort: election
protocol: TCP
- name: server
port: 2888
targetPort: server
protocol: TCP
selector:
app: zookeeper
release: zookeeper
---
# Source: zookeeper/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
name: zookeeper
labels:
app: zookeeper
release: zookeeper
spec:
type: ClusterIP
ports:
- name: client
port: 2281
protocol: TCP
targetPort: client
selector:
app: zookeeper
release: zookeeper
---
# Source: zookeeper/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: zookeeper
labels:
app: zookeeper
release: zookeeper
component: server
spec:
serviceName: zookeeper-headless
replicas: 3
selector:
matchLabels:
app: zookeeper
release: zookeeper
component: server
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: zookeeper
release: zookeeper
component: server
spec:
terminationGracePeriodSeconds: 1800
securityContext:
fsGroup: 1000
runAsUser: 1000
containers:
- name: zookeeper
image: "zookeeper:3.5.5"
imagePullPolicy: IfNotPresent
command:
- "/bin/bash"
- "-xec"
- "/config-scripts/run"
ports:
- name: client
containerPort: 2281
protocol: TCP
- name: election
containerPort: 3888
protocol: TCP
- name: server
containerPort: 2888
protocol: TCP
livenessProbe:
exec:
command:
- sh
- /config-scripts/ok
initialDelaySeconds: 20
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 2
successThreshold: 1
readinessProbe:
exec:
command:
- sh
- /config-scripts/ready
initialDelaySeconds: 20
periodSeconds: 30
timeoutSeconds: 5
failureThreshold: 2
successThreshold: 1
env:
- name: ZK_REPLICAS
value: "3"
- name: JMXAUTH
value: "false"
- name: JMXDISABLE
value: "false"
- name: JMXPORT
value: "1099"
- name: JMXSSL
value: "false"
- name: ZK_SYNC_LIMIT
value: "10"
- name: ZK_TICK_TIME
value: "2000"
- name: ZOO_AUTOPURGE_PURGEINTERVAL
value: "0"
- name: ZOO_AUTOPURGE_SNAPRETAINCOUNT
value: "3"
- name: ZOO_INIT_LIMIT
value: "5"
- name: ZOO_MAX_CLIENT_CNXNS
value: "60"
- name: ZOO_PORT
value: "2181"
- name: ZOO_STANDALONE_ENABLED
value: "false"
- name: ZOO_TICK_TIME
value: "2000"
resources:
{}
volumeMounts:
- name: data
mountPath: /data
- name: zookeeper-server-tls
mountPath: /tls/server
readOnly: true
- name: zookeeper-client-tls
mountPath: /tls/client
readOnly: true
- name: config
mountPath: /config-scripts
volumes:
- name: config
configMap:
name: zookeeper
defaultMode: 0555
- name: zookeeper-server-tls
secret:
secretName: zookeeper-server-tls
- name: zookeeper-client-tls
secret:
secretName: zookeeper-server-tls
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "5Gi"
View Git Blame Copy Permalink