openstack/openstack-helm-infra
Code Issues Proposed changes

(postgresql) Cert auth for replication connections

Browse Source 
- Change the Postgres configuration to use x509 client
  certs for authenticating the connections for replicating
  between Patroni nodes. This is a straightforward solution
  for support credential rotation for the replication user.
  Password authentication is problematic due to the declartive
  nature of helm charts and requiring an existing replication
  connection to replicate the rotated password.

Change-Id: I0c5456a01b3a36fee8ee4c986d25c4a1d807cb77
This commit is contained in:
Hussey, Scott (sh8121) 2019-06-04 14:48:53 -05:00 committed by Mahmoudi, Ahmad (am495p)
parent db164a2925
commit 9c27dd7576
6 changed files with 162 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
postgresql/templates/bin/_patroni_conversion.sh.tpl
View File

@ -25,7 +25,7 @@ limitations under the License.
#
# If any additional conversion steps are found to be needed, they can go here.
set -e
set -ex
function patroni_started() {
HOST=$1
@ -79,8 +79,10 @@ then
if [ ${USER_COUNT} -eq 0 ]; then
echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:"
${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \
WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';"
# CREATE ROLE defaults to NOLOGIN not to allow password based login.
# Replication user uses SSL Cert to connect.
${PSQL} -c "CREATE ROLE ${PATRONI_REPLICATION_USERNAME} \
WITH REPLICATION;"
echo "done."
else
echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do."

7
postgresql/templates/bin/_set_password.sh.tpl
View File

@ -29,7 +29,6 @@ cluster="$3"
PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }}
PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }}
PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }}
PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }}
if [[ x${role} == "xmaster" ]]; then
echo "I have become the patroni master: updating superuser and replication passwords"
@ -42,11 +41,5 @@ if [[ x${role} == "xmaster" ]]; then
echo "WARNING: Did not set superuser password!!!"
fi
if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then
psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE" -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';"
else
echo "WARNING: Did not set replication user password!!!"
fi
echo "password update complete"
fi

4
postgresql/templates/secret-replica.yaml
View File

@ -22,6 +22,6 @@ metadata:
name: {{ .Values.secrets.postgresql.replica }}
type: Opaque
data:
REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }}
REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }}
{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.replication "encode" true) | indent 2 }}
...
{{- end }}

25
postgresql/templates/secret-server.yaml Normal file
View File

@ -0,0 +1,25 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.secret_server }}
---
apiVersion: v1
kind: Secret
metadata:
name: {{ .Values.secrets.postgresql.server }}
type: Opaque
data:
{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.server "encode" true) | indent 2 }}
...
{{- end }}

105
postgresql/templates/statefulset.yaml
View File

@ -143,12 +143,59 @@ spec:
/bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }};
/bin/chmod 700 {{ .Values.storage.mount.path }};
/bin/chmod 700 {{ .Values.storage.mount.path }}/*;
/bin/cp {{ .Values.secrets.pki.client_cert_path }}_temp/* {{ .Values.secrets.pki.client_cert_path }}/.;
/bin/cp {{ .Values.secrets.pki.server_cert_path }}_temp/* {{ .Values.secrets.pki.server_cert_path }}/.;
/bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }};
/bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }}/*;
/bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }};
/bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }}/*;
/bin/chmod 700 {{ .Values.secrets.pki.client_cert_path }};
/bin/chmod 600 {{ .Values.secrets.pki.client_cert_path }}/*;
/bin/chmod 700 {{ .Values.secrets.pki.server_cert_path }};
/bin/chmod 600 {{ .Values.secrets.pki.server_cert_path }}/*;
{{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: postgresql-data
mountPath: {{ .Values.storage.mount.path }}
- name: server-certs
mountPath: {{ .Values.secrets.pki.server_cert_path }}
# server-cert-temp mountpoint is temp storage for secrets. We copy the
# secrets to server-certs folder and set owner and permissions.
# This is needed because the secrets are always created readonly.
- name: server-certs-temp
mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp
- name: postgresql-pki
subPath: crt
mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.crt
- name: postgresql-pki
subPath: key
mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.key
- name: replication-pki
subPath: ca
mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.crt
- name: replication-pki
subPath: caKey
mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.key
# client-certs is the permanent folder for the client secrets
- name: client-certs
mountPath: {{ .Values.secrets.pki.client_cert_path }}
# client-certs-temp is temporary folder for the client secrets, before they a copied to their permanent folder
- name: client-certs-temp
mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp
- name: replication-pki
subPath: crt
mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.crt
- name: replication-pki
subPath: key
mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.key
- name: postgresql-pki
subPath: ca
mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.crt
- name: postgresql-pki
subPath: caKey
mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.key
# This is for non-HA -> Patroni conversion and can be removed in the future
- name: patroni-conversion
{{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }}
@ -191,15 +238,7 @@ spec:
name: {{ .Values.secrets.postgresql.admin }}
key: 'POSTGRES_PASSWORD'
- name: PATRONI_REPLICATION_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql.replica }}
key: 'REPLICA_USER'
- name: PATRONI_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql.replica }}
key: 'REPLICA_PASSWORD'
value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }}
- name: PATRONI_RESTAPI_CONNECT_ADDRESS
value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- name: PATRONI_RESTAPI_LISTEN
@ -278,15 +317,7 @@ spec:
name: {{ .Values.secrets.postgresql.admin }}
key: 'POSTGRES_PASSWORD'
- name: PATRONI_REPLICATION_USERNAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql.replica }}
key: 'REPLICA_USER'
- name: PATRONI_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql.replica }}
key: 'REPLICA_PASSWORD'
value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }}
- name: PATRONI_RESTAPI_CONNECT_ADDRESS
value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
- name: PATRONI_RESTAPI_LISTEN
@ -299,6 +330,12 @@ spec:
value: $(PATRONI_SUPERUSER_PASSWORD)
- name: PATRONI_admin_OPTIONS
value: 'createrole,createdb'
- name: PGSSLROOTCERT
value: {{ .Values.secrets.pki.client_cert_path }}/ca.crt
- name: PGSSLCERT
value: "/home/postgres/.postgresql/postgresql.crt"
- name: PGSSLKEY
value: "/home/postgres/.postgresql/postgresql.key"
command:
- /tmp/start.sh
livenessProbe:
@ -338,9 +375,25 @@ spec:
readOnly: true
- name: postgresql-data
mountPath: {{ .Values.storage.mount.path }}
- name: server-certs
mountPath: {{ .Values.secrets.pki.server_cert_path }}
- name: client-certs
mountPath: {{ .Values.secrets.pki.client_cert_path }}
- name: postgres-home-config
mountPath: "/home/postgres/.postgresql"
- name: client-certs
subPath: "client.crt"
mountPath: "/home/postgres/.postgresql/postgresql.crt"
readOnly: true
- name: client-certs
subPath: "client.key"
mountPath: "/home/postgres/.postgresql/postgresql.key"
readOnly: true
volumes:
- name: pod-tmp
emptyDir: {}
- name: postgres-home-config
emptyDir: {}
- name: pg-run
emptyDir:
medium: "Memory"
@ -351,6 +404,22 @@ spec:
secret:
secretName: postgresql-bin
defaultMode: 0555
- name: client-certs-temp
emptyDir: {}
- name: server-certs-temp
emptyDir: {}
- name: client-certs
emptyDir: {}
- name: server-certs
emptyDir: {}
- name: replication-pki
secret:
secretName: {{ .Values.secrets.postgresql.replica }}
defaultMode: 0640
- name: postgresql-pki
secret:
secretName: {{ .Values.secrets.postgresql.server }}
defaultMode: 0640
- name: postgresql-etc
secret:
secretName: postgresql-etc

55
postgresql/values.yaml
View File

@ -30,6 +30,10 @@ pod:
server:
pod:
runAsUser: 999
# fsGroup used to allows cert file be witten to file.
fsGroup: 999
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
container:
set_volume_perms:
runAsUser: 0
@ -41,10 +45,6 @@ pod:
runAsUser: 999
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
pod:
runAsUser: 999
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
affinity:
anti:
type:
@ -253,6 +253,14 @@ conf:
max_replication_slots: 10
max_wal_senders: 10
max_worker_processes: 10
ssl: 'on'
# These relative paths are relative to data_dir
ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt
ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt
ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key
ssl_ciphers: 'HIGH:+3DES:!aNULL'
tcp_keepalives_idle: 900
tcp_keepalives_interval: 100
timezone: 'UTC'
track_commit_timestamp: 'on'
track_functions: all
@ -268,9 +276,8 @@ conf:
pg_hba:
- host all all 127.0.0.1/32 trust
- host all all 0.0.0.0/0 md5
- host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5
- host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
- local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
- hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1
- hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1
- local all all trust
postgresql:
{{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data,
@ -300,6 +307,14 @@ conf:
max_replication_slots: 10
max_wal_senders: 10
max_worker_processes: 10
ssl: 'on'
# These relative paths are relative to data_dir
ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt
ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt
ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key
ssl_ciphers: 'HIGH:+3DES:!aNULL'
tcp_keepalives_idle: 900
tcp_keepalives_interval: 100
timezone: 'UTC'
track_commit_timestamp: 'on'
track_functions: all
@ -309,9 +324,8 @@ conf:
pg_hba:
- host all all 127.0.0.1/32 trust
- host all all 0.0.0.0/0 md5
- host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5 # Fixes issue with Postgres 9.5
- host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
- local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
- hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1
- hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1
- local all all trust
watchdog:
mode: off # Allowed values: off, automatic, required
@ -322,9 +336,26 @@ conf:
pg_dumpall_options: null
secrets:
pki:
client_cert_path: /client_certs
server_cert_path: /server_certs
pod_cidr: 0.0.0.0/0
server:
hosts:
names:
# this name should be the service name for postgresql
- postgresql.ucp.svc.cluster.local
life: 365
replication:
hosts:
names:
# this name needs to be the same as endpoints.postgres.auth.replica.username
- standby
life: 365
postgresql:
admin: postgresql-admin
replica: postgresql-replication
replica: postgresql-replication-pki
server: postgresql-server-pki
exporter: postgresql-exporter
endpoints:
@ -348,7 +379,6 @@ endpoints:
password: password
replica:
username: standby
password: password
exporter:
username: psql_exporter
password: psql_exp_pass
@ -391,6 +421,7 @@ manifests:
job_image_repo_sync: true
secret_admin: true
secret_replica: true
secret_server: true
secret_etc: true
service: true
statefulset: true