From 9c27dd75763d420d9ffbe0abbda273065ab3f3dd Mon Sep 17 00:00:00 2001 From: "Hussey, Scott (sh8121)" Date: Tue, 4 Jun 2019 14:48:53 -0500 Subject: [PATCH] (postgresql) Cert auth for replication connections - Change the Postgres configuration to use x509 client certs for authenticating the connections for replicating between Patroni nodes. This is a straightforward solution for support credential rotation for the replication user. Password authentication is problematic due to the declartive nature of helm charts and requiring an existing replication connection to replicate the rotated password. Change-Id: I0c5456a01b3a36fee8ee4c986d25c4a1d807cb77 --- .../templates/bin/_patroni_conversion.sh.tpl | 8 +- postgresql/templates/bin/_set_password.sh.tpl | 7 -- postgresql/templates/secret-replica.yaml | 4 +- postgresql/templates/secret-server.yaml | 25 +++++ postgresql/templates/statefulset.yaml | 105 +++++++++++++++--- postgresql/values.yaml | 55 +++++++-- 6 files changed, 162 insertions(+), 42 deletions(-) create mode 100644 postgresql/templates/secret-server.yaml diff --git a/postgresql/templates/bin/_patroni_conversion.sh.tpl b/postgresql/templates/bin/_patroni_conversion.sh.tpl index 318ed4d08..8efa5c07c 100644 --- a/postgresql/templates/bin/_patroni_conversion.sh.tpl +++ b/postgresql/templates/bin/_patroni_conversion.sh.tpl @@ -25,7 +25,7 @@ limitations under the License. # # If any additional conversion steps are found to be needed, they can go here. -set -e +set -ex function patroni_started() { HOST=$1 @@ -79,8 +79,10 @@ then if [ ${USER_COUNT} -eq 0 ]; then echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:" - ${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \ - WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';" + # CREATE ROLE defaults to NOLOGIN not to allow password based login. + # Replication user uses SSL Cert to connect. + ${PSQL} -c "CREATE ROLE ${PATRONI_REPLICATION_USERNAME} \ + WITH REPLICATION;" echo "done." else echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do." diff --git a/postgresql/templates/bin/_set_password.sh.tpl b/postgresql/templates/bin/_set_password.sh.tpl index 3a6a45069..fae5e9f59 100644 --- a/postgresql/templates/bin/_set_password.sh.tpl +++ b/postgresql/templates/bin/_set_password.sh.tpl @@ -29,7 +29,6 @@ cluster="$3" PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }} PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }} PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }} -PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }} if [[ x${role} == "xmaster" ]]; then echo "I have become the patroni master: updating superuser and replication passwords" @@ -42,11 +41,5 @@ if [[ x${role} == "xmaster" ]]; then echo "WARNING: Did not set superuser password!!!" fi - if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then - psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';" - else - echo "WARNING: Did not set replication user password!!!" - fi - echo "password update complete" fi diff --git a/postgresql/templates/secret-replica.yaml b/postgresql/templates/secret-replica.yaml index 0c92b2008..03ac5867e 100644 --- a/postgresql/templates/secret-replica.yaml +++ b/postgresql/templates/secret-replica.yaml @@ -22,6 +22,6 @@ metadata: name: {{ .Values.secrets.postgresql.replica }} type: Opaque data: - REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }} - REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }} +{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.replication "encode" true) | indent 2 }} +... {{- end }} diff --git a/postgresql/templates/secret-server.yaml b/postgresql/templates/secret-server.yaml new file mode 100644 index 000000000..22b6c9a58 --- /dev/null +++ b/postgresql/templates/secret-server.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_server }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.secrets.postgresql.server }} +type: Opaque +data: +{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.server "encode" true) | indent 2 }} +... +{{- end }} diff --git a/postgresql/templates/statefulset.yaml b/postgresql/templates/statefulset.yaml index 8962adc8e..1ce8b94e9 100644 --- a/postgresql/templates/statefulset.yaml +++ b/postgresql/templates/statefulset.yaml @@ -143,12 +143,59 @@ spec: /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }}; /bin/chmod 700 {{ .Values.storage.mount.path }}; /bin/chmod 700 {{ .Values.storage.mount.path }}/*; + /bin/cp {{ .Values.secrets.pki.client_cert_path }}_temp/* {{ .Values.secrets.pki.client_cert_path }}/.; + /bin/cp {{ .Values.secrets.pki.server_cert_path }}_temp/* {{ .Values.secrets.pki.server_cert_path }}/.; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }}; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }}/*; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }}; + /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }}/*; + /bin/chmod 700 {{ .Values.secrets.pki.client_cert_path }}; + /bin/chmod 600 {{ .Values.secrets.pki.client_cert_path }}/*; + /bin/chmod 700 {{ .Values.secrets.pki.server_cert_path }}; + /bin/chmod 600 {{ .Values.secrets.pki.server_cert_path }}/*; {{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} volumeMounts: - name: pod-tmp mountPath: /tmp - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} + - name: server-certs + mountPath: {{ .Values.secrets.pki.server_cert_path }} + # server-cert-temp mountpoint is temp storage for secrets. We copy the + # secrets to server-certs folder and set owner and permissions. + # This is needed because the secrets are always created readonly. + - name: server-certs-temp + mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp + - name: postgresql-pki + subPath: crt + mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.crt + - name: postgresql-pki + subPath: key + mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.key + - name: replication-pki + subPath: ca + mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.crt + - name: replication-pki + subPath: caKey + mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.key + # client-certs is the permanent folder for the client secrets + - name: client-certs + mountPath: {{ .Values.secrets.pki.client_cert_path }} + # client-certs-temp is temporary folder for the client secrets, before they a copied to their permanent folder + - name: client-certs-temp + mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp + - name: replication-pki + subPath: crt + mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.crt + - name: replication-pki + subPath: key + mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.key + - name: postgresql-pki + subPath: ca + mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.crt + - name: postgresql-pki + subPath: caKey + mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.key # This is for non-HA -> Patroni conversion and can be removed in the future - name: patroni-conversion {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -191,15 +238,7 @@ spec: name: {{ .Values.secrets.postgresql.admin }} key: 'POSTGRES_PASSWORD' - name: PATRONI_REPLICATION_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.replica }} - key: 'REPLICA_USER' - - name: PATRONI_REPLICATION_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.replica }} - key: 'REPLICA_PASSWORD' + value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }} - name: PATRONI_RESTAPI_CONNECT_ADDRESS value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - name: PATRONI_RESTAPI_LISTEN @@ -278,15 +317,7 @@ spec: name: {{ .Values.secrets.postgresql.admin }} key: 'POSTGRES_PASSWORD' - name: PATRONI_REPLICATION_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.replica }} - key: 'REPLICA_USER' - - name: PATRONI_REPLICATION_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.secrets.postgresql.replica }} - key: 'REPLICA_PASSWORD' + value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }} - name: PATRONI_RESTAPI_CONNECT_ADDRESS value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - name: PATRONI_RESTAPI_LISTEN @@ -299,6 +330,12 @@ spec: value: $(PATRONI_SUPERUSER_PASSWORD) - name: PATRONI_admin_OPTIONS value: 'createrole,createdb' + - name: PGSSLROOTCERT + value: {{ .Values.secrets.pki.client_cert_path }}/ca.crt + - name: PGSSLCERT + value: "/home/postgres/.postgresql/postgresql.crt" + - name: PGSSLKEY + value: "/home/postgres/.postgresql/postgresql.key" command: - /tmp/start.sh livenessProbe: @@ -338,9 +375,25 @@ spec: readOnly: true - name: postgresql-data mountPath: {{ .Values.storage.mount.path }} + - name: server-certs + mountPath: {{ .Values.secrets.pki.server_cert_path }} + - name: client-certs + mountPath: {{ .Values.secrets.pki.client_cert_path }} + - name: postgres-home-config + mountPath: "/home/postgres/.postgresql" + - name: client-certs + subPath: "client.crt" + mountPath: "/home/postgres/.postgresql/postgresql.crt" + readOnly: true + - name: client-certs + subPath: "client.key" + mountPath: "/home/postgres/.postgresql/postgresql.key" + readOnly: true volumes: - name: pod-tmp emptyDir: {} + - name: postgres-home-config + emptyDir: {} - name: pg-run emptyDir: medium: "Memory" @@ -351,6 +404,22 @@ spec: secret: secretName: postgresql-bin defaultMode: 0555 + - name: client-certs-temp + emptyDir: {} + - name: server-certs-temp + emptyDir: {} + - name: client-certs + emptyDir: {} + - name: server-certs + emptyDir: {} + - name: replication-pki + secret: + secretName: {{ .Values.secrets.postgresql.replica }} + defaultMode: 0640 + - name: postgresql-pki + secret: + secretName: {{ .Values.secrets.postgresql.server }} + defaultMode: 0640 - name: postgresql-etc secret: secretName: postgresql-etc diff --git a/postgresql/values.yaml b/postgresql/values.yaml index b67362d63..6ee4381eb 100644 --- a/postgresql/values.yaml +++ b/postgresql/values.yaml @@ -30,6 +30,10 @@ pod: server: pod: runAsUser: 999 + # fsGroup used to allows cert file be witten to file. + fsGroup: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true container: set_volume_perms: runAsUser: 0 @@ -41,10 +45,6 @@ pod: runAsUser: 999 allowPrivilegeEscalation: false readOnlyRootFilesystem: true - pod: - runAsUser: 999 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true affinity: anti: type: @@ -253,6 +253,14 @@ conf: max_replication_slots: 10 max_wal_senders: 10 max_worker_processes: 10 + ssl: 'on' + # These relative paths are relative to data_dir + ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt + ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt + ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key + ssl_ciphers: 'HIGH:+3DES:!aNULL' + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 timezone: 'UTC' track_commit_timestamp: 'on' track_functions: all @@ -268,9 +276,8 @@ conf: pg_hba: - host all all 127.0.0.1/32 trust - host all all 0.0.0.0/0 md5 - - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 - - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 - - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1 + - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1 - local all all trust postgresql: {{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data, @@ -300,6 +307,14 @@ conf: max_replication_slots: 10 max_wal_senders: 10 max_worker_processes: 10 + ssl: 'on' + # These relative paths are relative to data_dir + ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt + ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt + ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key + ssl_ciphers: 'HIGH:+3DES:!aNULL' + tcp_keepalives_idle: 900 + tcp_keepalives_interval: 100 timezone: 'UTC' track_commit_timestamp: 'on' track_functions: all @@ -309,9 +324,8 @@ conf: pg_hba: - host all all 127.0.0.1/32 trust - host all all 0.0.0.0/0 md5 - - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5 - - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5 - - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5 + - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1 + - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1 - local all all trust watchdog: mode: off # Allowed values: off, automatic, required @@ -322,9 +336,26 @@ conf: pg_dumpall_options: null secrets: + pki: + client_cert_path: /client_certs + server_cert_path: /server_certs + pod_cidr: 0.0.0.0/0 + server: + hosts: + names: + # this name should be the service name for postgresql + - postgresql.ucp.svc.cluster.local + life: 365 + replication: + hosts: + names: + # this name needs to be the same as endpoints.postgres.auth.replica.username + - standby + life: 365 postgresql: admin: postgresql-admin - replica: postgresql-replication + replica: postgresql-replication-pki + server: postgresql-server-pki exporter: postgresql-exporter endpoints: @@ -348,7 +379,6 @@ endpoints: password: password replica: username: standby - password: password exporter: username: psql_exporter password: psql_exp_pass @@ -391,6 +421,7 @@ manifests: job_image_repo_sync: true secret_admin: true secret_replica: true + secret_server: true secret_etc: true service: true statefulset: true