Merge "(postgresql) Cert auth for replication connections"

postgresql/templates/bin/_patroni_conversion.sh.tpl

@@ -25,7 +25,7 @@ limitations under the License.
 #
 # If any additional conversion steps are found to be needed, they can go here.
 
-set -e
+set -ex
 
 function patroni_started() {
     HOST=$1
@@ -79,8 +79,10 @@ then
 
   if [ ${USER_COUNT} -eq 0 ]; then
     echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} doesn't exist yet; creating:"
-    ${PSQL} -c "CREATE USER ${PATRONI_REPLICATION_USERNAME} \
+    # CREATE ROLE defaults to NOLOGIN not to allow password based login.
-                WITH REPLICATION ENCRYPTED PASSWORD '${PATRONI_REPLICATION_PASSWORD}';"
+    # Replication user uses SSL Cert to connect.
+    ${PSQL} -c "CREATE ROLE ${PATRONI_REPLICATION_USERNAME} \
+                WITH REPLICATION;"
     echo "done."
   else
     echo "The patroni replication user ${PATRONI_REPLICATION_USERNAME} already exists: nothing to do."

postgresql/templates/bin/_set_password.sh.tpl

@@ -29,7 +29,6 @@ cluster="$3"
 PATRONI_SUPERUSER_USERNAME={{ .Values.endpoints.postgresql.auth.admin.username }}
 PATRONI_SUPERUSER_PASSWORD={{ .Values.endpoints.postgresql.auth.admin.password }}
 PATRONI_REPLICATION_USERNAME={{ .Values.endpoints.postgresql.auth.replica.username }}
-PATRONI_REPLICATION_PASSWORD={{ .Values.endpoints.postgresql.auth.replica.password }}
 
 if [[ x${role} == "xmaster" ]]; then
   echo "I have become the patroni master: updating superuser and replication passwords"
@@ -42,11 +41,5 @@ if [[ x${role} == "xmaster" ]]; then
     echo "WARNING: Did not set superuser password!!!"
   fi
 
-  if [[ ! -z "$PATRONI_REPLICATION_PASSWORD" && ! -z "$PATRONI_REPLICATION_USERNAME" ]]; then
-    psql -U $PATRONI_SUPERUSER_USERNAME -p "$PGPORT" -d "$PGDATABASE"  -c "ALTER ROLE $PATRONI_REPLICATION_USERNAME WITH PASSWORD '$PATRONI_REPLICATION_PASSWORD';"
-  else
-    echo "WARNING: Did not set replication user password!!!"
-  fi
-
   echo "password update complete"
 fi

postgresql/templates/secret-replica.yaml

@@ -22,6 +22,6 @@ metadata:
   name: {{ .Values.secrets.postgresql.replica }}
 type: Opaque
 data:
-  REPLICA_USER: {{ .Values.endpoints.postgresql.auth.replica.username | b64enc }}
+{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.replication "encode" true) | indent 2  }}
-  REPLICA_PASSWORD: {{ .Values.endpoints.postgresql.auth.replica.password | b64enc }}
+...
 {{- end }}

postgresql/templates/secret-server.yaml

@@ -0,0 +1,25 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_server }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secrets.postgresql.server }}
+type: Opaque
+data:
+{{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.secrets.pki.server "encode" true) | indent 2 }}
+...
+{{- end }}

postgresql/templates/statefulset.yaml

@@ -143,12 +143,59 @@ spec:
               /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.storage.mount.path }};
               /bin/chmod 700 {{ .Values.storage.mount.path }};
               /bin/chmod 700 {{ .Values.storage.mount.path }}/*;
+              /bin/cp {{ .Values.secrets.pki.client_cert_path }}_temp/* {{ .Values.secrets.pki.client_cert_path }}/.;
+              /bin/cp {{ .Values.secrets.pki.server_cert_path }}_temp/* {{ .Values.secrets.pki.server_cert_path }}/.;
+              /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }};
+              /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.client_cert_path }}/*;
+              /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }};
+              /bin/chown {{ .Values.pod.security_context.server.pod.runAsUser }} {{ .Values.secrets.pki.server_cert_path }}/*;
+              /bin/chmod 700 {{ .Values.secrets.pki.client_cert_path }};
+              /bin/chmod 600 {{ .Values.secrets.pki.client_cert_path }}/*;
+              /bin/chmod 700 {{ .Values.secrets.pki.server_cert_path }};
+              /bin/chmod 600 {{ .Values.secrets.pki.server_cert_path }}/*;
 {{ dict "envAll" $envAll "application" "server" "container" "set_volume_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
           volumeMounts:
             - name: pod-tmp
               mountPath: /tmp
             - name: postgresql-data
               mountPath: {{ .Values.storage.mount.path }}
+            - name: server-certs
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}
+              # server-cert-temp mountpoint is temp storage for secrets. We copy the
+              # secrets to server-certs folder and set owner and permissions.
+              # This is needed because the secrets are always created readonly.
+            - name: server-certs-temp
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp
+            - name: postgresql-pki
+              subPath: crt
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.crt
+            - name: postgresql-pki
+              subPath: key
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/server.key
+            - name: replication-pki
+              subPath: ca
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.crt
+            - name: replication-pki
+              subPath: caKey
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}_temp/ca.key
+              # client-certs is the permanent folder for the client secrets
+            - name: client-certs
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}
+              # client-certs-temp is temporary folder for the client secrets, before they a copied to their permanent folder
+            - name: client-certs-temp
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp
+            - name: replication-pki
+              subPath: crt
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.crt
+            - name: replication-pki
+              subPath: key
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/client.key
+            - name: postgresql-pki
+              subPath: ca
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.crt
+            - name: postgresql-pki
+              subPath: caKey
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}_temp/ca.key
         # This is for non-HA -> Patroni conversion and can be removed in the future
         - name: patroni-conversion
 {{ tuple $envAll "postgresql" | include "helm-toolkit.snippets.image" | indent 10 }}
@@ -191,15 +238,7 @@ spec:
                   name: {{ .Values.secrets.postgresql.admin }}
                   key: 'POSTGRES_PASSWORD'
             - name: PATRONI_REPLICATION_USERNAME
-              valueFrom:
+              value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }}
-                secretKeyRef:
-                  name: {{ .Values.secrets.postgresql.replica }}
-                  key: 'REPLICA_USER'
-            - name: PATRONI_REPLICATION_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.secrets.postgresql.replica }}
-                  key: 'REPLICA_PASSWORD'
             - name: PATRONI_RESTAPI_CONNECT_ADDRESS
               value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
             - name: PATRONI_RESTAPI_LISTEN
@@ -278,15 +317,7 @@ spec:
                   name: {{ .Values.secrets.postgresql.admin }}
                   key: 'POSTGRES_PASSWORD'
             - name: PATRONI_REPLICATION_USERNAME
-              valueFrom:
+              value: {{ index .Values.secrets.pki.replication.hosts.names 0 | quote }}
-                secretKeyRef:
-                  name: {{ .Values.secrets.postgresql.replica }}
-                  key: 'REPLICA_USER'
-            - name: PATRONI_REPLICATION_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.secrets.postgresql.replica }}
-                  key: 'REPLICA_PASSWORD'
             - name: PATRONI_RESTAPI_CONNECT_ADDRESS
               value: $(PATRONI_KUBERNETES_POD_IP):{{ tuple "postgresql-restapi" "internal" "restapi" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
             - name: PATRONI_RESTAPI_LISTEN
@@ -299,6 +330,12 @@ spec:
               value: $(PATRONI_SUPERUSER_PASSWORD)
             - name: PATRONI_admin_OPTIONS
               value: 'createrole,createdb'
+            - name: PGSSLROOTCERT
+              value: {{ .Values.secrets.pki.client_cert_path }}/ca.crt
+            - name: PGSSLCERT
+              value: "/home/postgres/.postgresql/postgresql.crt"
+            - name: PGSSLKEY
+              value: "/home/postgres/.postgresql/postgresql.key"
           command:
             - /tmp/start.sh
           livenessProbe:
@@ -338,9 +375,25 @@ spec:
               readOnly: true
             - name: postgresql-data
               mountPath: {{ .Values.storage.mount.path }}
+            - name: server-certs
+              mountPath: {{ .Values.secrets.pki.server_cert_path }}
+            - name: client-certs
+              mountPath: {{ .Values.secrets.pki.client_cert_path }}
+            - name: postgres-home-config
+              mountPath: "/home/postgres/.postgresql"
+            - name: client-certs
+              subPath: "client.crt"
+              mountPath: "/home/postgres/.postgresql/postgresql.crt"
+              readOnly: true
+            - name: client-certs
+              subPath: "client.key"
+              mountPath: "/home/postgres/.postgresql/postgresql.key"
+              readOnly: true
       volumes:
         - name: pod-tmp
           emptyDir: {}
+        - name: postgres-home-config
+          emptyDir: {}
         - name: pg-run
           emptyDir:
             medium: "Memory"
@@ -351,6 +404,22 @@ spec:
           secret:
             secretName: postgresql-bin
             defaultMode: 0555
+        - name: client-certs-temp
+          emptyDir: {}
+        - name: server-certs-temp
+          emptyDir: {}
+        - name: client-certs
+          emptyDir: {}
+        - name: server-certs
+          emptyDir: {}
+        - name: replication-pki
+          secret:
+            secretName: {{ .Values.secrets.postgresql.replica }}
+            defaultMode: 0640
+        - name: postgresql-pki
+          secret:
+            secretName: {{ .Values.secrets.postgresql.server }}
+            defaultMode: 0640
         - name: postgresql-etc
           secret:
             secretName: postgresql-etc

postgresql/values.yaml

@@ -30,6 +30,10 @@ pod:
     server:
       pod:
         runAsUser: 999
+        # fsGroup used to allows cert file be witten to file.
+        fsGroup: 999
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: true
       container:
         set_volume_perms:
           runAsUser: 0
@@ -41,10 +45,6 @@ pod:
           runAsUser: 999
           allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
-      pod:
-        runAsUser: 999
-        allowPrivilegeEscalation: false
-        readOnlyRootFilesystem: true
   affinity:
     anti:
       type:
@@ -253,6 +253,14 @@ conf:
             max_replication_slots: 10
             max_wal_senders: 10
             max_worker_processes: 10
+            ssl: 'on'
+            # These relative paths are relative to data_dir
+            ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt
+            ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt
+            ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key
+            ssl_ciphers: 'HIGH:+3DES:!aNULL'
+            tcp_keepalives_idle: 900
+            tcp_keepalives_interval: 100
             timezone: 'UTC'
             track_commit_timestamp: 'on'
             track_functions: all
@@ -268,9 +276,8 @@ conf:
       pg_hba:
       - host all all 127.0.0.1/32 trust
       - host all all 0.0.0.0/0 md5
-      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5  # Fixes issue with Postgres 9.5
+      - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1
-      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
+      - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1
-      - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
       - local all all trust
     postgresql:
       {{/* Note: the postgres pod mounts a volume at /var/lib/postgresql/data,
@@ -300,6 +307,14 @@ conf:
         max_replication_slots: 10
         max_wal_senders: 10
         max_worker_processes: 10
+        ssl: 'on'
+        # These relative paths are relative to data_dir
+        ssl_cert_file: {{ .Values.secrets.pki.server_cert_path }}/server.crt
+        ssl_ca_file: {{ .Values.secrets.pki.server_cert_path }}/ca.crt
+        ssl_key_file: {{ .Values.secrets.pki.server_cert_path }}/server.key
+        ssl_ciphers: 'HIGH:+3DES:!aNULL'
+        tcp_keepalives_idle: 900
+        tcp_keepalives_interval: 100
         timezone: 'UTC'
         track_commit_timestamp: 'on'
         track_functions: all
@@ -309,9 +324,8 @@ conf:
       pg_hba:
       - host all all 127.0.0.1/32 trust
       - host all all 0.0.0.0/0 md5
-      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 md5  # Fixes issue with Postgres 9.5
+      - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} {{ .Values.secrets.pki.pod_cidr }} cert clientcert=1
-      - host replication {{ .Values.endpoints.postgresql.auth.replica.username }} POD_IP_PATTERN/0 md5
+      - hostssl replication {{ .Values.endpoints.postgresql.auth.replica.username }} 127.0.0.1/32 cert clientcert=1
-      - local replication {{ .Values.endpoints.postgresql.auth.admin.username }} md5
       - local all all trust
     watchdog:
       mode: off  # Allowed values: off, automatic, required
@@ -322,9 +336,26 @@ conf:
     pg_dumpall_options: null
 
 secrets:
+  pki:
+    client_cert_path: /client_certs
+    server_cert_path: /server_certs
+    pod_cidr: 0.0.0.0/0
+    server:
+      hosts:
+        names:
+          # this name should be the service name for postgresql
+          - postgresql.ucp.svc.cluster.local
+      life: 365
+    replication:
+      hosts:
+        names:
+          # this name needs to be the same as endpoints.postgres.auth.replica.username
+          - standby
+      life: 365
   postgresql:
     admin: postgresql-admin
-    replica: postgresql-replication
+    replica: postgresql-replication-pki
+    server: postgresql-server-pki
     exporter: postgresql-exporter
 
 endpoints:
@@ -348,7 +379,6 @@ endpoints:
         password: password
       replica:
         username: standby
-        password: password
       exporter:
         username: psql_exporter
         password: psql_exp_pass
@@ -391,6 +421,7 @@ manifests:
   job_image_repo_sync: true
   secret_admin: true
   secret_replica: true
+  secret_server: true
   secret_etc: true
   service: true
   statefulset: true