From e954253a1a3d02f70279190f5e2479f9208f58d4 Mon Sep 17 00:00:00 2001
From: "Ritchie, Frank (fr801x)" <fr801x@att.com>
Date: Fri, 26 Feb 2021 10:12:15 -0500
Subject: [PATCH] Enable TLS for Ceph RGW

This PS is to optionally enable tls for ceph-rgw.

Change-Id: I4797ef41612143f8065ac8fec20ddeae2c0218a3
---
 ceph-rgw/Chart.yaml                         |  2 +-
 ceph-rgw/templates/bin/_helm-tests.sh.tpl   |  4 +++
 ceph-rgw/templates/bin/rgw/_init.sh.tpl     | 12 +++++++--
 ceph-rgw/templates/certificates.yaml        | 20 ++++++++++++++
 ceph-rgw/templates/deployment-rgw.yaml      | 29 ++++++++++++++-------
 ceph-rgw/templates/ingress-rgw.yaml         | 12 +++++++--
 ceph-rgw/templates/service-ingress-rgw.yaml |  7 ++++-
 ceph-rgw/templates/service-rgw.yaml         |  8 ++++--
 ceph-rgw/values.yaml                        |  7 ++++-
 ceph-rgw/values_overrides/tls.yaml          | 22 ++++++++++++++++
 release.asc                                 | 29 +++++++++++++++++++++
 releasenotes/notes/ceph-rgw.yaml            |  1 +
 12 files changed, 135 insertions(+), 18 deletions(-)
 create mode 100644 ceph-rgw/templates/certificates.yaml
 create mode 100644 ceph-rgw/values_overrides/tls.yaml
 create mode 100644 release.asc

diff --git a/ceph-rgw/Chart.yaml b/ceph-rgw/Chart.yaml
index 8e6b9ac74..b91b3d27d 100644
--- a/ceph-rgw/Chart.yaml
+++ b/ceph-rgw/Chart.yaml
@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph RadosGW
 name: ceph-rgw
-version: 0.1.4
+version: 0.1.5
 home: https://github.com/ceph/ceph
 ...
diff --git a/ceph-rgw/templates/bin/_helm-tests.sh.tpl b/ceph-rgw/templates/bin/_helm-tests.sh.tpl
index 505668f42..e1fff2971 100644
--- a/ceph-rgw/templates/bin/_helm-tests.sh.tpl
+++ b/ceph-rgw/templates/bin/_helm-tests.sh.tpl
@@ -83,7 +83,11 @@ function rgw_s3_bucket_validation ()
   echo "function: rgw_s3_bucket_validation"
 
   bucket=s3://rgw-test-bucket
+{{- if .Values.manifests.certificates }}
+  params="--host=$RGW_HOST --host-bucket=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-check-certificate"
+{{- else }}
   params="--host=$RGW_HOST --host-bucket=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-ssl"
+{{- end }}
 
   bucket_stat="$(s3cmd ls $params | grep ${bucket} || true)"
   if [[ -n "${bucket_stat}" ]]; then
diff --git a/ceph-rgw/templates/bin/rgw/_init.sh.tpl b/ceph-rgw/templates/bin/rgw/_init.sh.tpl
index b689d1516..66dc03e06 100644
--- a/ceph-rgw/templates/bin/rgw/_init.sh.tpl
+++ b/ceph-rgw/templates/bin/rgw/_init.sh.tpl
@@ -28,8 +28,12 @@ cat >> /etc/ceph/ceph.conf <<EOF
 {{ $key }} = {{ $value | quote  }}
 {{ end -}}
 {{- end -}}
-{{ if .Values.conf.rgw_ks.enabled }}
+{{- if .Values.conf.rgw_ks.enabled }}
+{{- if .Values.manifests.certificates }}
+rgw_frontends = "beast ssl_port=${RGW_FRONTEND_PORT} ssl_certificate=/etc/tls/tls.crt ssl_private_key=/etc/tls/tls.key"
+{{- else }}
 rgw_frontends = "civetweb port=${RGW_FRONTEND_PORT}"
+{{- end }}
 rgw_keystone_url = "${KEYSTONE_URL}"
 rgw_keystone_admin_user = "${OS_USERNAME}"
 rgw_keystone_admin_password = "${OS_PASSWORD}"
@@ -43,8 +47,12 @@ rgw_keystone_admin_domain = "${OS_USER_DOMAIN_NAME}"
 {{ end -}}
 {{- end -}}
 {{ end }}
-{{ if .Values.conf.rgw_s3.enabled }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{- if .Values.manifests.certificates }}
+rgw_frontends = "beast ssl_port=${RGW_FRONTEND_PORT} ssl_certificate=/etc/tls/tls.crt ssl_private_key=/etc/tls/tls.key"
+{{- else }}
 rgw_frontends = "beast port=${RGW_FRONTEND_PORT}"
+{{- end }}
 {{ range $key, $value := .Values.conf.rgw_s3.config -}}
 {{- if kindIs "slice" $value -}}
 {{ $key }} = {{ include "helm-toolkit.joinListWithComma" $value | quote }}
diff --git a/ceph-rgw/templates/certificates.yaml b/ceph-rgw/templates/certificates.yaml
new file mode 100644
index 000000000..06aca4475
--- /dev/null
+++ b/ceph-rgw/templates/certificates.yaml
@@ -0,0 +1,20 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+{{ $object_store_name := "object_store" }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{ $object_store_name = "ceph_object_store" }}
+{{- end }}
+{{- if .Values.manifests.certificates }}
+{{ dict "envAll" . "service" $object_store_name "type" "internal" | include "helm-toolkit.manifests.certificates" }}
+{{- end }}
diff --git a/ceph-rgw/templates/deployment-rgw.yaml b/ceph-rgw/templates/deployment-rgw.yaml
index 9a087e5b6..0849cc78d 100644
--- a/ceph-rgw/templates/deployment-rgw.yaml
+++ b/ceph-rgw/templates/deployment-rgw.yaml
@@ -15,6 +15,13 @@ limitations under the License.
 {{- if and .Values.manifests.deployment_rgw ( and .Values.deployment.ceph .Values.conf.features.rgw ) }}
 {{- $envAll := . }}
 
+{{ $object_store_name := "object_store" }}
+{{ $tls_secret := .Values.secrets.tls.object_store.api.internal | quote }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{ $object_store_name = "ceph_object_store" }}
+{{ $tls_secret = .Values.secrets.tls.ceph_object_store.api.internal | quote }}
+{{- end }}
+
 {{- $serviceAccountName := "ceph-rgw" }}
 
 {{- $_ := set $envAll.Values "__depParams" ( list ) }}
@@ -105,7 +112,7 @@ spec:
               value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }}
 {{ end }}
             - name: RGW_FRONTEND_PORT
-              value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
+              value: "{{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
           command:
             - /tmp/rgw-init.sh
           volumeMounts:
@@ -123,8 +130,8 @@ spec:
               mountPath: /etc/ceph/ceph.conf.template
               subPath: ceph.conf
               readOnly: true
-{{ if .Values.conf.rgw_ks.enabled }}
-{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.internal | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
+{{- if .Values.conf.rgw_ks.enabled }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.keystone | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
 {{- end }}
       containers:
         - name: ceph-rgw
@@ -135,21 +142,23 @@ spec:
             - name: CLUSTER
               value: "ceph"
             - name: RGW_FRONTEND_PORT
-              value: "{{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
+              value: "{{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}"
           command:
             - /tmp/rgw-start.sh
           ports:
-            - containerPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - containerPort: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           livenessProbe:
               httpGet:
                 path: /
-                port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+                port: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+                scheme: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
               initialDelaySeconds: 120
               timeoutSeconds: 5
           readinessProbe:
               httpGet:
                 path: /
-                port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+                port: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+                scheme: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
               timeoutSeconds: 5
           volumeMounts:
             - name: pod-tmp
@@ -173,6 +182,7 @@ spec:
             - name: pod-var-lib-ceph
               mountPath: /var/lib/ceph
               readOnly: false
+{{- dict "enabled" .Values.manifests.certificates "name" $tls_secret "path" "/etc/tls" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
       volumes:
         - name: pod-tmp
           emptyDir: {}
@@ -194,7 +204,8 @@ spec:
         - name: ceph-bootstrap-rgw-keyring
           secret:
             secretName: {{ .Values.secrets.keyrings.rgw }}
-{{ if .Values.conf.rgw_ks.enabled }}
-{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- dict "enabled" .Values.manifests.certificates "name" $tls_secret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
+{{- if .Values.conf.rgw_ks.enabled }}
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.keystone | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
 {{- end }}
 {{- end }}
diff --git a/ceph-rgw/templates/ingress-rgw.yaml b/ceph-rgw/templates/ingress-rgw.yaml
index b212ad63b..bddef8480 100644
--- a/ceph-rgw/templates/ingress-rgw.yaml
+++ b/ceph-rgw/templates/ingress-rgw.yaml
@@ -12,7 +12,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- if and .Values.manifests.ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }}
-{{- $ingressOpts := dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" -}}
+{{- if and .Values.manifests.ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw) ) }}
+{{- $ingressOpts := dict "envAll" . "backendServiceType" "object_store"  "backendPort" "ceph-rgw" -}}
+{{- if .Values.manifests.certificates }}
+{{- if .Values.conf.rgw_ks.enabled }}
+{{- $ingressOpts = dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" "certIssuer" .Values.endpoints.object_store.host_fqdn_override.default.tls.issuerRef.name -}}
+{{- end }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{- $ingressOpts = dict "envAll" . "backendServiceType" "ceph_object_store" "backendPort" "ceph-rgw" "certIssuer" .Values.endpoints.ceph_object_store.host_fqdn_override.default.tls.issuerRef.name -}}
+{{- end }}
+{{- end }}
 {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}
diff --git a/ceph-rgw/templates/service-ingress-rgw.yaml b/ceph-rgw/templates/service-ingress-rgw.yaml
index e995c5c66..9a9bd4d60 100644
--- a/ceph-rgw/templates/service-ingress-rgw.yaml
+++ b/ceph-rgw/templates/service-ingress-rgw.yaml
@@ -12,7 +12,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+{{ $object_store_name := "object_store" }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{ $object_store_name = "ceph_object_store" }}
+{{- end }}
+
 {{- if and .Values.manifests.service_ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }}
-{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "object_store" -}}
+{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" $object_store_name -}}
 {{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
 {{- end }}
diff --git a/ceph-rgw/templates/service-rgw.yaml b/ceph-rgw/templates/service-rgw.yaml
index 33a1d928c..fcb236ef9 100644
--- a/ceph-rgw/templates/service-rgw.yaml
+++ b/ceph-rgw/templates/service-rgw.yaml
@@ -14,6 +14,10 @@ limitations under the License.
 
 {{- if and .Values.manifests.service_rgw ( and .Values.deployment.ceph .Values.conf.features.rgw ) }}
 {{- $envAll := . }}
+{{ $object_store_name := "object_store" }}
+{{- if .Values.conf.rgw_s3.enabled }}
+{{ $object_store_name = "ceph_object_store" }}
+{{- end }}
 ---
 apiVersion: v1
 kind: Service
@@ -22,9 +26,9 @@ metadata:
 spec:
   ports:
   - name: ceph-rgw
-    port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    port: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     protocol: TCP
-    targetPort: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    targetPort: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
   {{ if .Values.network.api.node_port.enabled }}
     nodePort: {{ .Values.network.api.node_port.port }}
   {{ end }}
diff --git a/ceph-rgw/values.yaml b/ceph-rgw/values.yaml
index b9e29265e..8f9d17f83 100644
--- a/ceph-rgw/values.yaml
+++ b/ceph-rgw/values.yaml
@@ -262,7 +262,12 @@ secrets:
     object_store:
       api:
         public: ceph-tls-public
-        internal: keystone-tls-api
+        internal: ceph-rgw-ks-tls-api
+        keystone: keystone-tls-api
+    ceph_object_store:
+      api:
+        public: ceph-rgw-s3-tls-public
+        internal: ceph-rgw-s3-tls-api
 
 network:
   api:
diff --git a/ceph-rgw/values_overrides/tls.yaml b/ceph-rgw/values_overrides/tls.yaml
new file mode 100644
index 000000000..cf02f8842
--- /dev/null
+++ b/ceph-rgw/values_overrides/tls.yaml
@@ -0,0 +1,22 @@
+---
+endpoints:
+  object_store:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: ceph-rgw-ks-tls-api
+          issuerRef:
+            name: ca-clusterissuer
+            kind: ClusterIssuer
+  ceph_object_store:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: ceph-rgw-s3-tls-api
+          issuerRef:
+            name: ca-clusterissuer
+            kind: ClusterIssuer
+
+manifests:
+  certificates: true
+...
diff --git a/release.asc b/release.asc
new file mode 100644
index 000000000..d2961c52e
--- /dev/null
+++ b/release.asc
@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBFX4hgkBEADLqn6O+UFp+ZuwccNldwvh5PzEwKUPlXKPLjQfXlQRig1flpCH
+E0HJ5wgGlCtYd3Ol9f9+qU24kDNzfbs5bud58BeE7zFaZ4s0JMOMuVm7p8JhsvkU
+C/Lo/7NFh25e4kgJpjvnwua7c2YrA44ggRb1QT19ueOZLK5wCQ1mR+0GdrcHRCLr
+7Sdw1d7aLxMT+5nvqfzsmbDullsWOD6RnMdcqhOxZZvpay8OeuK+yb8FVQ4sOIzB
+FiNi5cNOFFHg+8dZQoDrK3BpwNxYdGHsYIwU9u6DWWqXybBnB9jd2pve9PlzQUbO
+eHEa4Z+jPqxY829f4ldaql7ig8e6BaInTfs2wPnHJ+606g2UH86QUmrVAjVzlLCm
+nqoGymoAPGA4ObHu9X3kO8viMBId9FzooVqR8a9En7ZE0Dm9O7puzXR7A1f5sHoz
+JdYHnr32I+B8iOixhDUtxIY4GA8biGATNaPd8XR2Ca1hPuZRVuIiGG9HDqUEtXhV
+fY5qjTjaThIVKtYgEkWMT+Wet3DPPiWT3ftNOE907e6EWEBCHgsEuuZnAbku1GgD
+LBH4/a/yo9bNvGZKRaTUM/1TXhM5XgVKjd07B4cChgKypAVHvef3HKfCG2U/DkyA
+LjteHt/V807MtSlQyYaXUTGtDCrQPSlMK5TjmqUnDwy6Qdq8dtWN3DtBWQARAQAB
+tCpDZXBoLmNvbSAocmVsZWFzZSBrZXkpIDxzZWN1cml0eUBjZXBoLmNvbT6JAjgE
+EwECACIFAlX4hgkCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEOhKwsBG
+DzmUXdIQAI8YPcZMBWdv489q8CzxlfRIRZ3Gv/G/8CH+EOExcmkVZ89mVHngCdAP
+DOYCl8twWXC1lwJuLDBtkUOHXNuR5+Jcl5zFOUyldq1Hv8u03vjnGT7lLJkJoqpG
+l9QD8nBqRvBU7EM+CU7kP8+09b+088pULil+8x46PwgXkvOQwfVKSOr740Q4J4nm
+/nUOyTNtToYntmt2fAVWDTIuyPpAqA6jcqSOC7Xoz9cYxkVWnYMLBUySXmSS0uxl
+3p+wK0lMG0my/gb+alke5PAQjcE5dtXYzCn+8Lj0uSfCk8Gy0ZOK2oiUjaCGYN6D
+u72qDRFBnR3jaoFqi03bGBIMnglGuAPyBZiI7LJgzuT9xumjKTJW3kN4YJxMNYu1
+FzmIyFZpyvZ7930vB2UpCOiIaRdZiX4Z6ZN2frD3a/vBxBNqiNh/BO+Dex+PDfI4
+TqwF8zlcjt4XZ2teQ8nNMR/D8oiYTUW8hwR4laEmDy7ASxe0p5aijmUApWq5UTsF
++s/QbwugccU0iR5orksM5u9MZH4J/mFGKzOltfGXNLYI6D5Mtwrnyi0BsF5eY0u6
+vkdivtdqrq2DXY+ftuqLOQ7b+t1RctbcMHGPptlxFuN9ufP5TiTWSpfqDwmHCLsT
+k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA
+=/Tod
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/releasenotes/notes/ceph-rgw.yaml b/releasenotes/notes/ceph-rgw.yaml
index 23c3c3e69..fdf5764aa 100644
--- a/releasenotes/notes/ceph-rgw.yaml
+++ b/releasenotes/notes/ceph-rgw.yaml
@@ -5,4 +5,5 @@ ceph-rgw:
   - 0.1.2 Uplift from Nautilus to Octopus release
   - 0.1.3 update rbac api version
   - 0.1.4 Rgw placement target support
+  - 0.1.5 Add tls support
 ...