From 9efb353b83c59e891b1b85dc6567044de0f5ac17 Mon Sep 17 00:00:00 2001 From: Doug Aaser Date: Mon, 18 Mar 2019 18:08:17 +0000 Subject: [PATCH] Patroni inclusion work for HA Postgres This patchset aims to add HA Clustering support for Postgres. HA Clustering provides automatic failover in the event of the database going down in addition to keeping replicas of the database for rebuilding in the event of a node going down. To achieve this clustering we use [Patroni](https://github.com/zalando/patroni) which offers HA clustering support for Postgres. Patroni is a daemon that runs in the background and keeps track of which node in your cluster is currently the leader node and routes all traffic on the Postgresql endpoint to that node. If the leader node goes down, Patroni holds an election to chose a new leader and updates the endpoint to route traffic accordingly. All communication between nodes is done by a Patroni created endpoint, seperate from the externally facing Postgres endpoint. Note that, although the postgresql helm chart can be upgraded from non-patroni to patroni clustering, the previous `postgresql` endpoints object (which is not directly managed by helm) must be deleted via an out-of-band mechanism so that it may be replaced by the patroni-managed endpoints. If Postgres itself is leveraged for the deployment process, this must be done with careful timing. Note that the old endpoints had a port named "db", and the new endpoints has a port named "postgresql". - Picking up patchset: https://review.openstack.org/#/c/591663 Co-authored-by: Tony Sorrentino Co-authored-by: Randeep Jalli Co-authored-by: Pete Birley Co-authored-by: Matt McEuen Change-Id: I721b745017dc1ea7ae05dfd9f8d5dd08d0965985 --- .../templates/bin/_patroni_conversion.sh.tpl | 121 ++++++++ postgresql/templates/bin/_readiness.sh.tpl | 8 +- postgresql/templates/bin/_set_password.sh.tpl | 52 ++++ postgresql/templates/bin/_start.sh.tpl | 73 +++-- postgresql/templates/configmap-bin.yaml | 24 +- postgresql/templates/configmap-etc.yaml | 28 ++ .../templates/cron-job-backup-postgres.yaml | 6 +- postgresql/templates/pod-test.yaml | 4 +- postgresql/templates/secret-replica.yaml | 27 ++ .../{service.yaml => service-postgres.yaml} | 6 +- postgresql/templates/service-restapi.yaml | 30 ++ postgresql/templates/statefulset.yaml | 266 ++++++++++++++++-- postgresql/values.yaml | 168 ++++++++++- .../osh-infra-monitoring/130-postgresql.sh | 4 +- 14 files changed, 748 insertions(+), 69 deletions(-) create mode 100644 postgresql/templates/bin/_patroni_conversion.sh.tpl create mode 100644 postgresql/templates/bin/_set_password.sh.tpl create mode 100644 postgresql/templates/configmap-etc.yaml create mode 100644 postgresql/templates/secret-replica.yaml rename postgresql/templates/{service.yaml => service-postgres.yaml} (82%) create mode 100644 postgresql/templates/service-restapi.yaml diff --git a/postgresql/templates/bin/_patroni_conversion.sh.tpl b/postgresql/templates/bin/_patroni_conversion.sh.tpl new file mode 100644 index 000000000..318ed4d08 --- /dev/null +++ b/postgresql/templates/bin/_patroni_conversion.sh.tpl @@ -0,0 +1,121 @@ +#!/bin/bash + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This script creates the patroni replication user if it doesn't exist. +# This is only needed for brownfield upgrade scenarios, on top of sites that +# were greenfield-deployed with a pre-patroni version of postgres. +# +# For greenfield deployments, the patroni-enabled postgresql chart will +# create this user automatically. +# +# If any additional conversion steps are found to be needed, they can go here. + +set -e + +function patroni_started() { + HOST=$1 + PORT=$2 + STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT}; + echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3; + cat <&3 | tail -n1 | grep -o \"running\"") + + [[ x${STATUS} == "xrunning" ]] +} + +PGDATABASE=${PGDATABASE:-'postgres'} +PGHOST=${PGHOST:-'127.0.0.1'} +PGPORT={{- tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +PSQL="psql -h ${PGHOST} -p ${PGPORT} -d ${PGDATABASE}" + +PVC_MNT={{- .Values.storage.mount.path }} +FILE_MADE_BY_POSTGRES=${PVC_MNT}/pgdata/pg_xlog +FILE_MADE_BY_PATRONI=${PVC_MNT}/pgdata/patroni.dynamic.json + +TIMEOUT=0 + +# Only need to add the user once, on the first replica +if [ "x${POD_NAME}" != "xpostgresql-0" ]; then + echo "Nothing to do on ${POD_NAME}" + exit 0 +fi + +# Look for a file-based clue that we're migrating from vanilla pg to patroni. +# This is lighter-weight than checking in the database for the user, since +# we have to fire up the database at this point to do the check. +if [[ -e "${FILE_MADE_BY_POSTGRES}" && ! -e "${FILE_MADE_BY_PATRONI}" ]] +then + echo "We are upgrading to Patroni -- checking for replication user" + + # Fire up a temporary postgres + /docker-entrypoint.sh postgres & + while ! $PSQL -c "select 1;"; do + sleep 1 + if [[ $TIMEOUT -gt 120 ]]; then + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done + TIMEOUT=0 + + # Add the replication user if it doesn't exist + USER_COUNT=$(${PSQL} -qt -c \ + "SELECT COUNT(*) FROM pg_roles \ + WHERE rolname='${PATRONI_REPLICATION_USERNAME}'") + + if [ ${USER_COUNT} -eq 0 ]; then + echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:" + ${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \ + WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';" + echo "done." + else + echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do." + fi + + # Start Patroni to assimilate the postgres + sed "s/POD_IP_PATTERN/${PATRONI_KUBERNETES_POD_IP}/g" \ + /tmp/patroni-templated.yaml > /tmp/patroni.yaml + + READY_FLAG="i am the leader with the lock" + PATRONI_LOG=/tmp/patroni_conversion.log + /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni-templated.yaml &> ${PATRONI_LOG} & + + # Sleep until patroni is running + while ! grep -q "${READY_FLAG}" ${PATRONI_LOG}; do + sleep 5 + if [[ $TIMEOUT -gt 24 ]]; then + echo "A timeout occurred. Patroni logs:" + cat ${PATRONI_LOG} + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done + TIMEOUT=0 + + # Gracefully stop postgres and patroni + while pkill INT --uid postgres; do + sleep 5 + if [[ $TIMEOUT -gt 24 ]]; then + echo "A timeout occurred. Patroni logs:" + cat ${PATRONI_LOG} + exit 1 + fi + TIMEOUT=$((TIMEOUT+1)) + done +else + echo "Patroni is already in place: nothing to do." +fi diff --git a/postgresql/templates/bin/_readiness.sh.tpl b/postgresql/templates/bin/_readiness.sh.tpl index c8c6b269d..7c48fafad 100644 --- a/postgresql/templates/bin/_readiness.sh.tpl +++ b/postgresql/templates/bin/_readiness.sh.tpl @@ -1,7 +1,7 @@ #!/usr/bin/env bash {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,4 +18,8 @@ limitations under the License. set -ex -pg_isready -U ${POSTGRES_USER} +if [ -f /tmp/postgres-disable-liveness-probe ]; then + exit 0 +else + pg_isready -U ${PATRONI_SUPERUSER_USERNAME} +fi diff --git a/postgresql/templates/bin/_set_password.sh.tpl b/postgresql/templates/bin/_set_password.sh.tpl new file mode 100644 index 000000000..3a6a45069 --- /dev/null +++ b/postgresql/templates/bin/_set_password.sh.tpl @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +PGDATABASE=${PGDATABASE:-'postgres'} +PGHOST=${PGHOST:-'127.0.0.1'} +PGPORT={{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + +# These are passed in via the Patroni callback interface +action="$1" +role="$2" +cluster="$3" + +# Note: this script when rendered is stored in a secret and encrypted to disk. +PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }} +PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }} +PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }} +PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }} + +if [[ x${role} == "xmaster" ]]; then + echo "I have become the patroni master: updating superuser and replication passwords" + + # It can take a few seconds for a freshly promoted leader to become read/write. + sleep 10 + if [[ ! -z "$PATRONI_SUPERUSER_PASSWORD" && ! -z "$PATRONI_SUPERUSER_USERNAME" ]]; then + psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_SUPERUSER_USERNAME WITH PASSWORD '$PATRONI_SUPERUSER_PASSWORD';" + else + echo "WARNING: Did not set superuser password!!!" + fi + + if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then + psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';" + else + echo "WARNING: Did not set replication user password!!!" + fi + + echo "password update complete" +fi diff --git a/postgresql/templates/bin/_start.sh.tpl b/postgresql/templates/bin/_start.sh.tpl index e173caadc..2cd2edc3c 100644 --- a/postgresql/templates/bin/_start.sh.tpl +++ b/postgresql/templates/bin/_start.sh.tpl @@ -1,7 +1,7 @@ #!/usr/bin/env bash {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,25 +16,62 @@ See the License for the specific language governing permissions and limitations under the License. */}} -# Disable echo mode while setting the password -# unless we are in debug mode -{{- if .Values.conf.debug }} -set -x -{{- end }} -set -e +set -ex -POSTGRES_DB=${POSTGRES_DB:-"postgres"} +function patroni_started() { + HOST=$1 + PORT=$2 + STATUS=$(timeout 10 bash -c "exec 3<>/dev/tcp/${HOST}/${PORT}; + echo -e \"GET / HTTP/1.1\r\nConnection: close\r\n\" >&3; + cat <&3 | tail -n1 | grep -o \"running\"") -# Check if the Postgres data directory exists before attempting to -# set the password -if [[ -d "$PGDATA" && -s "$PGDATA/PG_VERSION" ]] -then - postgres --single -D "$PGDATA" "$POSTGRES_DB" < \ + /tmp/patroni.yaml + +FILE_MADE_BY_PATRONI=${PGDATA}/patroni.dynamic.json +if [[ ! $POD_NAME -eq "postgresql-0" ]]; then + + echo "I am not postgresql pod zero: disabling liveness probe temporarily" + # disable liveness probe as it may take some time for the pod to come online + touch /tmp/postgres-disable-liveness-probe + + # During normal upgrades, we just need to turn liveness probes off temporarily + # for the sake of password rotation - need to bounce all pods at once + # (overriding RollingUpdate) to avoid deadlock. This accounts for that. + sleep 60 + + # During initial bootstrapping, we need to sequence 0,1,2 + if [[ ! -e "${FILE_MADE_BY_PATRONI}" ]]; then + echo "patroni has not been initialized on this node" + # NOTE: this boolean forces a second check after a delay. This accounts for a + # scenario during initial vanilla postgres -> patroni conversion, where + # a temporary master is brought up, killed off, and then restarted. + # This can be safely removed in the future, once all clusters are converted. + WAITED_EXTRA="false" + + while [ ${WAITED_EXTRA} = "false" ]; do + while ! patroni_started "${SVC_FQDN}" "${SVC_PORT}"; do + echo "Waiting until a Leader is elected..." + sleep 5 + done + # See note above: this code can be removed once all clusters are Patroni. + if [ ${WAITED_EXTRA} = "false" ]; then + echo "Leader is up; sleeping to ensure it gets through restarts..." + sleep 10 + WAITED_EXTRA="true" + fi + done + fi + + rm -fv /tmp/postgres-disable-liveness-probe fi -set -x - -exec /docker-entrypoint.sh postgres -N {{ .Values.conf.postgresql.max_connections | quote }} -B {{ .Values.conf.postgresql.shared_buffers | quote }} +exec /usr/bin/python3 /usr/local/bin/patroni /tmp/patroni.yaml diff --git a/postgresql/templates/configmap-bin.yaml b/postgresql/templates/configmap-bin.yaml index bae75be0f..f5c931ea5 100644 --- a/postgresql/templates/configmap-bin.yaml +++ b/postgresql/templates/configmap-bin.yaml @@ -19,24 +19,22 @@ limitations under the License. {{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} --- apiVersion: v1 -kind: ConfigMap +{{/* Note: this is a secret because credentials must be rendered into the password script. */}} +kind: Secret metadata: name: postgresql-bin +type: Opaque data: {{- if .Values.images.local_registry.active }} - image-repo-sync.sh: | -{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + image-repo-sync.sh: {{- include "helm-toolkit.scripts.image_repo_sync" . | b64enc }} {{- end }} - start.sh: | -{{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - readiness.sh: | -{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - db_test.sh: | -{{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start.sh: {{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + readiness.sh: {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + db_test.sh: {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- if .Values.conf.backup.enabled }} - backup_postgresql.sh: | -{{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - restore_postgresql.sh: | -{{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_postgresql.sh: {{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + restore_postgresql.sh: {{ tuple "bin/_restore_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- end }} + set_password.sh: {{ tuple "bin/_set_password.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + patroni_conversion.sh: {{ tuple "bin/_patroni_conversion.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }} {{- end }} diff --git a/postgresql/templates/configmap-etc.yaml b/postgresql/templates/configmap-etc.yaml new file mode 100644 index 000000000..9dddf06a5 --- /dev/null +++ b/postgresql/templates/configmap-etc.yaml @@ -0,0 +1,28 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgresql-etc +type: Opaque +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.patroni "key" "patroni.yaml" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/postgresql/templates/cron-job-backup-postgres.yaml b/postgresql/templates/cron-job-backup-postgres.yaml index aefb37774..1014c4f84 100644 --- a/postgresql/templates/cron-job-backup-postgres.yaml +++ b/postgresql/templates/cron-job-backup-postgres.yaml @@ -96,10 +96,10 @@ spec: secret: secretName: postgresql-secrets defaultMode: 0600 - - configMap: + - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 365 - name: postgresql-bin - name: postgresql-bin {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} - name: postgresql-backup-dir persistentVolumeClaim: diff --git a/postgresql/templates/pod-test.yaml b/postgresql/templates/pod-test.yaml index d260f32a9..45ed8d436 100644 --- a/postgresql/templates/pod-test.yaml +++ b/postgresql/templates/pod-test.yaml @@ -70,8 +70,8 @@ spec: - name: pod-tmp emptyDir: {} - name: postgresql-bin - configMap: - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 0555 ... {{- end }} diff --git a/postgresql/templates/secret-replica.yaml b/postgresql/templates/secret-replica.yaml new file mode 100644 index 000000000..0c92b2008 --- /dev/null +++ b/postgresql/templates/secret-replica.yaml @@ -0,0 +1,27 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_replica }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.postgresql.replica }} +type: Opaque +data: + REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }} + REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }} +{{- end }} diff --git a/postgresql/templates/service.yaml b/postgresql/templates/service-postgres.yaml similarity index 82% rename from postgresql/templates/service.yaml rename to postgresql/templates/service-postgres.yaml index 7ad24b288..31d0195a9 100644 --- a/postgresql/templates/service.yaml +++ b/postgresql/templates/service-postgres.yaml @@ -1,5 +1,5 @@ {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,8 +23,6 @@ metadata: name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: db + - name: postgresql port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - selector: -{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} {{- end }} diff --git a/postgresql/templates/service-restapi.yaml b/postgresql/templates/service-restapi.yaml new file mode 100644 index 000000000..36dbc3f14 --- /dev/null +++ b/postgresql/templates/service-restapi.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "postgresql-restapi" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: restapi + port: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- end }} diff --git a/postgresql/templates/statefulset.yaml b/postgresql/templates/statefulset.yaml index 673ad16a7..3dfb8c85c 100644 --- a/postgresql/templates/statefulset.yaml +++ b/postgresql/templates/statefulset.yaml @@ -1,5 +1,5 @@ {{/* -Copyright 2017 The Openstack-Helm Authors. +Copyright 2019 The Openstack-Helm Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,6 +20,81 @@ limitations under the License. {{- $serviceAccountName := "postgresql" }} {{ tuple $envAll "postgresql" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch + # delete and deletecollection are required only for 'patronictl remove' + - delete + - deletecollection + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - patch + - update + # the following three privileges are necessary only when using endpoints + - create + - list + - watch + # delete and deletecollection are required only for 'patronictl remove' + - delete + - deletecollection + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + # The following privilege is only necessary for creation of headless service + # for postgresql-config endpoint, in order to prevent cleaning it up by the + # k8s master. + - apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - list + - patch + - update + - watch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -28,19 +103,27 @@ metadata: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} labels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: serviceName: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" replicas: {{ .Values.pod.replicas.server }} selector: matchLabels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} template: metadata: labels: {{ tuple $envAll "postgresql" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + cluster-name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-admin-hash: {{ tuple "secret-admin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-replica-hash: {{ tuple "secret-replica.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-secrets-etc-hash: {{ tuple "secrets-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} {{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} @@ -53,56 +136,193 @@ spec: - name: set-volume-perms {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - "/bin/chown" - - {{ .Values.pod.security_context.server.pod.runAsUser | quote }} - - {{ .Values.storage.mount.path | quote }} + command: ["/bin/sh", "-c"] + args: + - set -xe; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }}; + /bin/chmod 700 {{ .Values.storage.mount.path }}; + /bin/chmod 700 {{ .Values.storage.mount.path }}/*; {{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} - subPath: {{ .Values.storage.mount.subpath }} + # This is for non-HA -> Patroni conversion and can be removed in the future + - name: patroni-conversion +{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: PGDATA + value: "{{ .Values.storage.mount.path }}/pgdata" + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PATRONI_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_SUPERUSER_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_USER' + - name: PATRONI_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_PASSWORD' + - name: PATRONI_REPLICATION_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_USER' + - name: PATRONI_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_PASSWORD' + - name: PATRONI_RESTAPI_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_RESTAPI_LISTEN + value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_LISTEN + value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_admin_PASSWORD + value: $(PATRONI_SUPERUSER_PASSWORD) + - name: PATRONI_admin_OPTIONS + value: 'createrole,createdb' + command: + - /tmp/patroni_conversion.sh +{{ dict "envAll" $envAll "application" "server" "container" "patroni_conversion" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: patroni-conversion-tmp + mountPath: /var/run/postgresql + - name: postgresql-bin + mountPath: /tmp/patroni_conversion.sh + subPath: patroni_conversion.sh + readOnly: true + - name: postgresql-data + mountPath: {{ .Values.storage.mount.path }} + - name: postgresql-etc + mountPath: /tmp/patroni-templated.yaml + subPath: patroni.yaml + readOnly: true containers: - name: postgresql {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ dict "envAll" $envAll "application" "server" "container" "postgresql" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} ports: + - containerPort: {{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP - containerPort: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP env: - - name: 'POSTGRES_PASSWORD' + - name: PGDATA + value: "{{ .Values.storage.mount.path }}/pgdata" + - name: PATRONI_KUBERNETES_POD_IP valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.admin }} - key: 'POSTGRES_PASSWORD' - - name: 'POSTGRES_USER' + fieldRef: + fieldPath: status.podIP + - name: PATRONI_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PATRONI_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PATRONI_KUBERNETES_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: PATRONI_SUPERUSER_USERNAME valueFrom: secretKeyRef: name: {{ .Values.secrets.postgresql.admin }} key: 'POSTGRES_USER' - - name: 'PGDATA' - value: {{ .Values.storage.mount.path | quote }} + - name: PATRONI_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.admin }} + key: 'POSTGRES_PASSWORD' + - name: PATRONI_REPLICATION_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_USER' + - name: PATRONI_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.postgresql.replica }} + key: 'REPLICA_PASSWORD' + - name: PATRONI_RESTAPI_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_RESTAPI_LISTEN + value: 0.0.0.0:{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_CONNECT_ADDRESS + value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_POSTGRESQL_LISTEN + value: 0.0.0.0:{{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: PATRONI_admin_PASSWORD + value: $(PATRONI_SUPERUSER_PASSWORD) + - name: PATRONI_admin_OPTIONS + value: 'createrole,createdb' command: - /tmp/start.sh livenessProbe: exec: command: - /tmp/readiness.sh - initialDelaySeconds: 20 + initialDelaySeconds: 30 timeoutSeconds: 5 + failureThreshold: 10 readinessProbe: exec: command: - /tmp/readiness.sh - initialDelaySeconds: 20 + initialDelaySeconds: 30 timeoutSeconds: 5 + failureThreshold: 10 volumeMounts: - name: pod-tmp mountPath: /tmp - name: pg-run mountPath: /var/run/postgresql + - name: postgresql-bin + mountPath: /tmp/set_password.sh + subPath: set_password.sh + readOnly: true - name: postgresql-bin mountPath: /tmp/start.sh subPath: start.sh @@ -111,19 +331,29 @@ spec: mountPath: /tmp/readiness.sh subPath: readiness.sh readOnly: true + - name: postgresql-etc + mountPath: /tmp/patroni-templated.yaml + subPath: patroni.yaml + readOnly: true - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} - subPath: {{ .Values.storage.mount.subpath }} volumes: - name: pod-tmp emptyDir: {} - name: pg-run emptyDir: medium: "Memory" + # This is for non-HA -> Patroni conversion and can be removed in the future + - name: patroni-conversion-tmp + emptyDir: {} - name: postgresql-bin - configMap: - name: postgresql-bin + secret: + secretName: postgresql-bin defaultMode: 0555 + - name: postgresql-etc + secret: + secretName: postgresql-etc + defaultMode: 0444 {{- if not .Values.storage.pvc.enabled }} - name: postgresql-data hostPath: diff --git a/postgresql/values.yaml b/postgresql/values.yaml index 4b4485a0d..8dfa0ec77 100644 --- a/postgresql/values.yaml +++ b/postgresql/values.yaml @@ -1,4 +1,4 @@ -# Copyright 2017 The Openstack-Helm Authors. +# Copyright 2019 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +37,14 @@ pod: postgresql: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + patroni_conversion: + runAsUser: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + pod: + runAsUser: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true affinity: anti: type: @@ -46,8 +54,7 @@ pod: weight: default: 10 replicas: - #only 1 replica currently supported - server: 1 + server: 3 prometheus_postgresql_exporter: 1 lifecycle: upgrades: @@ -106,10 +113,10 @@ pod: memory: "1024Mi" cpu: "2000m" -# using dockerhub postgresql: https://hub.docker.com/r/library/postgres/tags/ +# using dockerhub patroni: https://hub.docker.com/r/openstackhelm/patroni/tags/ images: tags: - postgresql: "docker.io/postgres:9.5" + postgresql: "docker.io/openstackhelm/patroni:latest-ubuntu_xenial" dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 image_repo_sync: docker.io/docker:17.07.0 prometheus_postgresql_exporter: docker.io/wrouesnel/postgres_exporter:v0.4.6 @@ -131,8 +138,8 @@ storage: host: host_path: /data/openstack-helm/postgresql mount: - path: /var/lib/postgresql/data - subpath: pgdata + path: /var/lib/postgresql + subpath: . labels: server: @@ -205,8 +212,137 @@ jobs: conf: debug: false postgresql: - max_connections: 100 shared_buffers: 128MB + max_connections: 100 + patroni: | + scope: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + kubernetes: + labels: + application: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + component: server + use_endpoints: true + ports: + - name: {{ tuple "postgresql" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + port: {{ tuple "postgresql" "internal" "postgresql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + bootstrap: + dcs: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + postgresql: + data_dir: '{{ .Values.storage.mount.path }}/pgdata' + pgpass: '{{ .Values.storage.mount.path }}/pgpass' + use_pg_rewind: true + parameters: + archive_mode: 'on' + archive_timeout: 1800s + autovacuum_analyze_scale_factor: 0.02 + autovacuum_max_workers: 5 + autovacuum_vacuum_scale_factor: 0.05 + checkpoint_completion_target: 0.9 + datestyle: 'iso, mdy' + default_text_search_config: 'pg_catalog.english' + external_pid_file: '/tmp/postgres.pid' + hot_standby: 'on' + lc_messages: 'en_US.utf8' + lc_monetary: 'en_US.utf8' + lc_numeric: 'en_US.utf8' + lc_time: 'en_US.utf8' + log_autovacuum_min_duration: 0 + log_checkpoints: 'on' + log_connections: 'on' + log_disconnections: 'on' + log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m ' + log_lock_waits: 'on' + log_min_duration_statement: 500 + log_statement: none + log_temp_files: 0 + log_timezone: 'UTC' + max_connections: {{ .Values.conf.postgresql.max_connections }} + max_replication_slots: 10 + max_wal_senders: 10 + max_worker_processes: 10 + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 + timezone: 'UTC' + track_commit_timestamp: 'on' + track_functions: all + wal_keep_segments: 100 + wal_level: 'logical' + wal_log_hints: 'on' + initdb: + - auth-host: md5 + - auth-local: trust + - encoding: UTF8 + - locale: en_US.UTF-8 + - data-checksums + pg_hba: + - host all all 127.0.0.1/32 trust + - host all all 0.0.0.0/0 md5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 + - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - local all all trust + postgresql: + {{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data, + so let's just avoid it and use /var/lib/postgresql/pgdata instead. + Patroni moves this directory to a backup under the parent directory + (/var/lib/postgresql) under certain failure recovery scenarios, so + /var/lib/postgres itself must be exposed to the pod as a pvc mount.*/}} + data_dir: '{{ .Values.storage.mount.path }}/pgdata' + pgpass: '{{ .Values.storage.mount.path }}/pgpass' + callbacks: + on_role_change: /tmp/set_password.sh + on_start: /tmp/set_password.sh + use_pg_rewind: true + parameters: + archive_mode: 'on' + archive_timeout: 1800s + autovacuum_analyze_scale_factor: 0.02 + autovacuum_max_workers: 5 + autovacuum_vacuum_scale_factor: 0.05 + checkpoint_completion_target: 0.9 + datestyle: 'iso, mdy' + default_text_search_config: 'pg_catalog.english' + external_pid_file: '/tmp/postgres.pid' + hot_standby: 'on' + lc_messages: 'en_US.utf8' + lc_monetary: 'en_US.utf8' + lc_numeric: 'en_US.utf8' + lc_time: 'en_US.utf8' + log_autovacuum_min_duration: 0 + log_checkpoints: 'on' + log_connections: 'on' + log_disconnections: 'on' + log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m ' + log_lock_waits: 'on' + log_min_duration_statement: 500 + log_statement: none + log_temp_files: 0 + log_timezone: 'UTC' + max_connections: {{ .Values.conf.postgresql.max_connections }} + max_replication_slots: 10 + max_wal_senders: 10 + max_worker_processes: 10 + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 + timezone: 'UTC' + track_commit_timestamp: 'on' + track_functions: all + shared_buffers: {{ .Values.conf.postgresql.shared_buffers }} + wal_keep_segments: 100 + wal_level: 'logical' + wal_log_hints: 'on' + pg_hba: + - host all all 127.0.0.1/32 trust + - host all all 0.0.0.0/0 md5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 + - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 + - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - local all all trust + watchdog: + mode: off # Allowed values: off, automatic, required backup: enabled: true base_path: /var/backup @@ -216,6 +352,7 @@ conf: secrets: postgresql: admin: postgresql-admin + replica: postgresql-replication exporter: postgresql-exporter endpoints: @@ -237,6 +374,9 @@ endpoints: admin: username: postgres password: password + replica: + username: standby + password: password exporter: username: psql_exporter password: psql_exp_pass @@ -249,6 +389,16 @@ endpoints: port: postgresql: default: 5432 + postgresql_restapi: + hosts: + default: postgresql-restapi + host_fqdn_override: + default: null + path: null + scheme: postgresql + port: + restapi: + default: 8008 prometheus_postgresql_exporter: namespace: null hosts: @@ -265,8 +415,10 @@ endpoints: manifests: configmap_bin: true + configmap_etc: true job_image_repo_sync: true secret_admin: true + secret_replica: true secret_etc: true service: true statefulset: true diff --git a/tools/deployment/osh-infra-monitoring/130-postgresql.sh b/tools/deployment/osh-infra-monitoring/130-postgresql.sh index d99351593..c4dd70b0e 100755 --- a/tools/deployment/osh-infra-monitoring/130-postgresql.sh +++ b/tools/deployment/osh-infra-monitoring/130-postgresql.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2017 The Openstack-Helm Authors. +# Copyright 2019 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -25,6 +25,8 @@ helm upgrade --install postgresql ./postgresql \ --namespace=osh-infra \ --set monitoring.prometheus.enabled=true \ --set storage.pvc.size=1Gi \ + --set storage.pvc.enabled=true \ + --set pod.replicas.server=3 \ ${OSH_INFRA_EXTRA_HELM_ARGS} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB}