Split out RGW from ceph client charts

Co-Authored-By: Chinasubbareddy Mallavarapu<cr3938@att.com>

Change-Id: I8b091493bb79c7eab525fbaf7801028398772c5d
Signed-off-by: Pete Birley <pete@port.direct>

ceph-client/templates/bin/_init-dirs.sh.tpl

@@ -20,27 +20,22 @@ set -ex
 export LC_ALL=C
 : "${HOSTNAME:=$(uname -n)}"
 : "${MGR_NAME:=${HOSTNAME}}"
-: "${RGW_NAME:=${HOSTNAME}}"
 : "${MDS_NAME:=mds-${HOSTNAME}}"
 : "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}"
-: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}"
 : "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}"
 
-for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ${RGW_BOOTSTRAP_KEYRING}; do
+for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING}; do
   mkdir -p "$(dirname "$keyring")"
 done
 
 # Let's create the ceph directories
-for DIRECTORY in mds radosgw tmp mgr; do
+for DIRECTORY in mds tmp mgr; do
   mkdir -p "/var/lib/ceph/${DIRECTORY}"
 done
 
 # Create socket directory
 mkdir -p /run/ceph
 
-# Creating rados directories
-mkdir -p "/var/lib/ceph/radosgw/${RGW_NAME}"
-
 # Create the MDS directory
 mkdir -p "/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}"
 

ceph-client/templates/configmap-bin.yaml

@@ -48,9 +48,4 @@ data:
   mgr-check.sh: |
 {{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 
-  rgw-start.sh: |
-{{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  rgw-init-keystone.sh: |
-{{ tuple "bin/rgw/_init_keystone.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-
 {{- end }}

ceph-client/templates/configmap-etc-client.yaml

@@ -39,10 +39,6 @@ limitations under the License.
 {{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
 {{- end -}}
 
-{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}}
-{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}}
-{{- end -}}
-
 ---
 apiVersion: v1
 kind: ConfigMap

ceph-client/values.yaml

@@ -19,7 +19,6 @@
 
 deployment:
   ceph: true
-  rgw_keystone_user_and_endpoints: false
 
 release_group: null
 
@@ -31,11 +30,7 @@ images:
     ceph_mds: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_mgr: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     ceph_rbd_pool: 'docker.io/port/ceph-config-helper:v1.10.3'
-    ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
     dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
-    ks_endpoints: 'docker.io/openstackhelm/heat:newton'
-    ks_service: 'docker.io/openstackhelm/heat:newton'
-    ks_user: 'docker.io/openstackhelm/heat:newton'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
     active: false
@@ -53,9 +48,6 @@ labels:
   mds:
     node_selector_key: ceph-mds
     node_selector_value: enabled
-  rgw:
-    node_selector_key: ceph-rgw
-    node_selector_value: enabled
   mgr:
     node_selector_key: ceph-mgr
     node_selector_value: enabled
@@ -65,7 +57,6 @@ pod:
   replicas:
     mds: 2
     mgr: 2
-    rgw: 2
   affinity:
     anti:
       type:
@@ -81,13 +72,6 @@ pod:
       limits:
         memory: "50Mi"
         cpu: "500m"
-    rgw:
-      requests:
-        memory: "5Mi"
-        cpu: "250m"
-      limits:
-        memory: "50Mi"
-        cpu: "500m"
     mgr:
       requests:
         memory: "5Mi"
@@ -103,27 +87,6 @@ pod:
         requests:
           memory: "128Mi"
           cpu: "500m"
-      ks_endpoints:
-        requests:
-          memory: "128Mi"
-          cpu: "100m"
-        limits:
-          memory: "1024Mi"
-          cpu: "2000m"
-      ks_service:
-        requests:
-          memory: "128Mi"
-          cpu: "100m"
-        limits:
-          memory: "1024Mi"
-          cpu: "2000m"
-      ks_user:
-        requests:
-          memory: "128Mi"
-          cpu: "100m"
-        limits:
-          memory: "1024Mi"
-          cpu: "2000m"
       image_repo_sync:
         requests:
           memory: "128Mi"
@@ -140,37 +103,14 @@ secrets:
     rgw: ceph-bootstrap-rgw-keyring
     mgr: ceph-bootstrap-mgr-keyring
     admin: ceph-client-admin-keyring
-  identity:
-    admin: ceph-keystone-admin
-    swift: ceph-keystone-user
-    user_rgw: ceph-keystone-user-rgw
-  tls:
-    object_store:
-      api:
-        public: ceph-tls-public
 
 network:
-  api:
-    ingress:
-      public: true
-      classes:
-        namespace: "nginx"
-        cluster: "nginx-cluster"
-      annotations:
-        nginx.ingress.kubernetes.io/rewrite-target: /
-        nginx.ingress.kubernetes.io/proxy-body-size: "0"
-        nginx.org/proxy-max-temp-file-size: "0"
-    external_policy_local: false
-    node_port:
-      enabled: false
-      port: 30004
   public: 192.168.0.0/16
   cluster: 192.168.0.0/16
 
 conf:
   features:
     mds: true
-    rgw: true
     mgr: true
   pool:
   #NOTE(portdirect): this drives a simple approximation of
@@ -269,18 +209,6 @@ conf:
         application: rgw
         replication: 3
         percent_total_data: 34.8
-  rgw_ks:
-    enabled: false
-    config:
-      rgw_keystone_api_version: 3
-      rgw_keystone_accepted_roles: "admin, member"
-      rgw_keystone_implicit_tenants: true
-      rgw_keystone_make_new_tenants: true
-      rgw_s3_auth_use_keystone: true
-      rgw_swift_account_in_url: true
-      rgw_swift_url: null
-      #NOTE (portdirect): See http://tracker.ceph.com/issues/21226
-      rgw_keystone_token_cache_size: 0
   ceph:
     global:
       # auth
@@ -318,20 +246,6 @@ dependencies:
       services:
         - endpoint: internal
           service: ceph_mon
-    ks_endpoints:
-      jobs:
-        - ceph-ks-service
-      services:
-        - endpoint: internal
-          service: identity
-    ks_service:
-      services:
-        - endpoint: internal
-          service: identity
-    ks_user:
-      services:
-        - endpoint: internal
-          service: identity
     mds:
       jobs:
         - ceph-storage-keys-generator
@@ -361,14 +275,6 @@ dependencies:
       services:
         - endpoint: internal
           service: ceph_mon
-    rgw:
-      jobs:
-        - ceph-storage-keys-generator
-        - ceph-rgw-keyring-generator
-        - ceph-rbd-pool
-      services:
-        - endpoint: internal
-          service: ceph_mon
     image_repo_sync:
       services:
         - endpoint: internal
@@ -425,61 +331,6 @@ endpoints:
     port:
       registry:
         node: 5000
-  identity:
-    name: keystone
-    namespace: null
-    auth:
-      admin:
-        region_name: RegionOne
-        username: admin
-        password: password
-        project_name: admin
-        user_domain_name: default
-        project_domain_name: default
-      swift:
-        role: admin
-        region_name: RegionOne
-        username: swift
-        password: password
-        project_name: service
-        user_domain_name: service
-        project_domain_name: service
-    hosts:
-      default: keystone-api
-      public: keystone
-    host_fqdn_override:
-      default: null
-    path:
-      default: /v3
-    scheme:
-      default: http
-    port:
-      api:
-        default: 80
-        internal: 5000
-  object_store:
-    name: swift
-    namespace: null
-    hosts:
-      default: ceph-rgw
-      public: radosgw
-    host_fqdn_override:
-      default: null
-      # NOTE(portdirect): this chart supports TLS for fqdn over-ridden public
-      # endpoints using the following format:
-      # public:
-      #   host: null
-      #   tls:
-      #     crt: null
-      #     key: null
-    path:
-      default: /swift/v1/KEY_$(tenant_id)s
-    scheme:
-      default: http
-    port:
-      api:
-        default: 8088
-        public: 80
   ceph_mon:
     namespace: null
     hosts:
@@ -513,22 +364,11 @@ monitoring:
 
 manifests:
   configmap_bin: true
-  configmap_bin_ks: true
   configmap_etc: true
   deployment_mds: true
-  deployment_rgw: true
   deployment_mgr: true
-  ingress_rgw: true
   job_bootstrap: false
   job_cephfs_client_key: true
   job_image_repo_sync: true
-  job_ks_endpoints: true
-  job_ks_service: true
-  job_ks_user: true
   job_rbd_pool: true
-  secret_ingress_tls: true
-  secret_keystone_rgw: true
-  secret_keystone: true
-  service_ingress_rgw: true
   service_mgr: true
-  service_rgw: true

ceph-rgw/Chart.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+description: OpenStack-Helm Ceph RadosGW
+name: ceph-rgw
+version: 0.1.0

ceph-rgw/requirements.yaml

@@ -0,0 +1,18 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dependencies:
+  - name: helm-toolkit
+    repository: http://localhost:8879/charts
+    version: 0.1.0

ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+export HOME=/tmp
+
+cat <<EOF > /etc/ceph/ceph.client.admin.keyring
+[client.admin]
+    key = $(cat /tmp/client-keyring)
+EOF
+
+exit 0

ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -x
+if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then
+  SECRET=$(mktemp --suffix .yaml)
+  KEYRING=$(mktemp --suffix .keyring)
+  function cleanup {
+      rm -f ${SECRET} ${KEYRING}
+  }
+  trap cleanup EXIT
+fi
+
+function kube_ceph_keyring_gen () {
+  CEPH_KEY=$1
+  CEPH_KEY_TEMPLATE=$2
+  sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" /tmp/ceph-templates/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n'
+}
+
+set -ex
+if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then
+  ceph -s
+  if USERINFO=$(ceph auth get client.bootstrap-rgw); then
+    KEYSTR=$(echo $USERINFO | sed 's/.*\( key = .*\) caps mon.*/\1/')
+    echo $KEYSTR  > ${KEYRING}
+  else
+    #NOTE(Portdirect): Determine proper privs to assign keyring
+    ceph auth get-or-create client.bootstrap-rgw \
+      mon "allow profile bootstrap-rgw" \
+      -o ${KEYRING}
+  fi
+  FINAL_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING})
+  cat > ${SECRET} <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "os-ceph-bootstrap-rgw-keyring"
+type: Opaque
+data:
+ ceph.keyring: $( kube_ceph_keyring_gen ${FINAL_KEYRING} "bootstrap.keyring.rgw"  )
+EOF
+  kubectl apply --namespace ${NAMESPACE} -f ${SECRET}
+
+fi

ceph-rgw/templates/bin/_init-dirs.sh.tpl

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+export LC_ALL=C
+: "${HOSTNAME:=$(uname -n)}"
+: "${RGW_NAME:=${HOSTNAME}}"
+: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}"
+
+for keyring in ${RGW_BOOTSTRAP_KEYRING}; do
+  mkdir -p "$(dirname "$keyring")"
+done
+
+# Let's create the ceph directories
+for DIRECTORY in radosgw tmp; do
+  mkdir -p "/var/lib/ceph/${DIRECTORY}"
+done
+
+# Create socket directory
+mkdir -p /run/ceph
+
+# Creating rados directories
+mkdir -p "/var/lib/ceph/radosgw/${RGW_NAME}"
+
+# Adjust the owner of all those directories
+chown -R ceph. /run/ceph/ /var/lib/ceph/*

ceph-rgw/templates/configmap-bin-ks.yaml

@@ -20,7 +20,7 @@ limitations under the License.
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ceph-client-bin-ks
+  name: ceph-rgw-bin-ks
 data:
   ks-service.sh: |
 {{- include "helm-toolkit.scripts.keystone_service" . | indent 4 }}

ceph-rgw/templates/configmap-bin.yaml

@@ -0,0 +1,42 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-rgw-bin
+data:
+{{- if .Values.images.local_registry.active }}
+  image-repo-sync.sh: |
+{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }}
+{{- end }}
+
+  init-dirs.sh: |
+{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+
+  rgw-start.sh: |
+{{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  rgw-init-keystone.sh: |
+{{ tuple "bin/rgw/_init_keystone.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  storage-init.sh: |
+{{ tuple "bin/_ceph-rgw-storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-admin-keyring.sh: |
+{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+
+{{- end }}

ceph-rgw/templates/configmap-ceph-rgw-templates.yaml

@@ -0,0 +1,27 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.configmap_ceph_templates .Values.manifests.job_ceph_rgw_storage_init }}
+{{- $envAll := . }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-templates
+data:
+  bootstrap.keyring.rgw: |
+{{ .Values.conf.templates.keyring.bootstrap.rgw | indent 4 }}
+{{- end }}

ceph-rgw/templates/configmap-etc-client.yaml

@@ -0,0 +1,55 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "ceph.configmap.etc" }}
+{{- $configMapName := index . 0 }}
+{{- $envAll := index . 1 }}
+{{- with $envAll }}
+
+{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
+
+{{- if empty .Values.conf.ceph.global.mon_host -}}
+{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
+{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.cluster_network -}}
+{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.ceph.osd.public_network -}}
+{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}}
+{{- end -}}
+
+{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}}
+{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}}
+{{- end -}}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $configMapName }}
+data:
+  ceph.conf: |
+{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }}
+
+{{- end }}
+{{- end }}
+{{- end }}
+{{- if .Values.manifests.configmap_etc }}
+{{- list "ceph-rgw-etc" . | include "ceph.configmap.etc" }}
+{{- end }}

ceph-rgw/templates/deployment-rgw.yaml

@@ -51,7 +51,7 @@ spec:
             - name: CLUSTER
               value: "ceph"
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-rgw-bin
               mountPath: /tmp/init-dirs.sh
               subPath: init-dirs.sh
               readOnly: true
@@ -85,11 +85,11 @@ spec:
           volumeMounts:
             - name: pod-etc-ceph
               mountPath: /etc/ceph
-            - name: ceph-client-bin
+            - name: ceph-rgw-bin
               mountPath: /tmp/rgw-init-keystone.sh
               subPath: rgw-init-keystone.sh
               readOnly: true
-            - name: ceph-client-etc
+            - name: ceph-rgw-etc
               mountPath: /tmp/ceph.conf
               subPath: ceph.conf
               readOnly: true
@@ -119,22 +119,18 @@ spec:
                 port: {{ tuple "object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
               timeoutSeconds: 5
           volumeMounts:
-            - name: ceph-client-bin
+            - name: ceph-rgw-bin
               mountPath: /tmp/rgw-start.sh
               subPath: rgw-start.sh
               readOnly: true
             - name: pod-etc-ceph
               mountPath: /etc/ceph
 {{- if not .Values.conf.rgw_ks.enabled }}
-            - name: ceph-client-etc
+            - name: ceph-rgw-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
 {{- end }}
-            - name: ceph-client-admin-keyring
-              mountPath: /etc/ceph/ceph.client.admin.keyring
-              subPath: ceph.client.admin.keyring
-              readOnly: true
             - name: ceph-bootstrap-rgw-keyring
               mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
               subPath: ceph.keyring
@@ -148,22 +144,19 @@ spec:
       volumes:
         - name: pod-etc-ceph
           emptyDir: {}
-        - name: ceph-client-bin
+        - name: ceph-rgw-bin
           configMap:
-            name: ceph-client-bin
+            name: ceph-rgw-bin
             defaultMode: 0555
-        - name: ceph-client-etc
+        - name: ceph-rgw-etc
           configMap:
-            name: ceph-client-etc
+            name: ceph-rgw-etc
             defaultMode: 0444
         - name: pod-var-lib-ceph
           emptyDir: {}
         - name: pod-run
           emptyDir:
             medium: "Memory"
-        - name: ceph-client-admin-keyring
-          secret:
-            secretName: {{ .Values.secrets.keyrings.admin }}
         - name: ceph-bootstrap-rgw-keyring
           secret:
             secretName: {{ .Values.secrets.keyrings.rgw }}

ceph-rgw/templates/job-ks-endpoints.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.job_ks_endpoints .Values.deployment.rgw_keystone_user_and_endpoints }}
-{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}}
+{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}}
 {{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_endpoints" }}
 {{- end }}

ceph-rgw/templates/job-ks-service.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.job_ks_service .Values.deployment.rgw_keystone_user_and_endpoints }}
-{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}}
+{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}}
 {{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_service" }}
 {{- end }}

ceph-rgw/templates/job-ks-user.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.job_ks_user .Values.deployment.rgw_keystone_user_and_endpoints }}
-{{- $ksUserJob := dict "envAll" . "configMapBin" "ceph-client-bin-ks" "serviceName" "ceph" "serviceUser" "swift" -}}
+{{- $ksUserJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceUser" "swift" -}}
 {{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }}
 {{- end }}

ceph-rgw/templates/job-rgw-storage-init.yaml

@@ -0,0 +1,133 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.job_ceph_rgw_storage_init }}
+{{- $envAll := . }}
+
+{{- $serviceAccountName := "ceph-rgw-storage-init" }}
+{{ tuple $envAll "rgw_storage_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+      - update
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-rgw-storage-init
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "ceph-rgw" "rgw-storage-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+      serviceAccountName: {{ $serviceAccountName }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
+      initContainers:
+{{ tuple $envAll "rgw_storage_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        - name: ceph-keyring-placement
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-admin-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-rgw-bin
+              mountPath: /tmp/ceph-admin-keyring.sh
+              subPath: ceph-admin-keyring.sh
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+      containers:
+        - name: ceph-rgw-storage-init
+{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          env:
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: STORAGE_BACKEND
+              value: "ceph-rgw"
+          command:
+            - /tmp/storage-init.sh
+          volumeMounts:
+            - name: ceph-rgw-bin
+              mountPath: /tmp/storage-init.sh
+              subPath: storage-init.sh
+              readOnly: true
+            - name: ceph-templates
+              mountPath: /tmp/ceph-templates
+              readOnly: true
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+      volumes:
+        - name: ceph-rgw-bin
+          configMap:
+            name: ceph-rgw-bin
+            defaultMode: 0555
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+            defaultMode: 0444
+        - name: ceph-templates
+          configMap:
+            name: ceph-templates
+            defaultMode: 0444
+        - name: ceph-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin | quote }}
+{{- end }}

ceph-rgw/values.yaml

@@ -0,0 +1,432 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default values for ceph-client.
+# This is a YAML-formatted file.
+# Declare name/value pairs to be passed into your templates.
+# name: value
+
+deployment:
+  ceph: false
+  rgw_keystone_user_and_endpoints: false
+
+release_group: null
+
+images:
+  pull_policy: IfNotPresent
+  tags:
+    ceph_config_helper: 'docker.io/port/ceph-config-helper:v1.10.3'
+    ceph_rgw: 'docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04'
+    dep_check: 'quay.io/stackanetes/kubernetes-entrypoint:v0.3.1'
+    image_repo_sync: docker.io/docker:17.07.0
+    ks_endpoints: 'docker.io/openstackhelm/heat:newton'
+    ks_service: 'docker.io/openstackhelm/heat:newton'
+    ks_user: 'docker.io/openstackhelm/heat:newton'
+  local_registry:
+    active: false
+    exclude:
+      - dep_check
+      - image_repo_sync
+
+labels:
+  job:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
+  rgw:
+    node_selector_key: ceph-rgw
+    node_selector_value: enabled
+
+pod:
+  dns_policy: "ClusterFirstWithHostNet"
+  replicas:
+    rgw: 2
+  affinity:
+    anti:
+      type:
+        default: preferredDuringSchedulingIgnoredDuringExecution
+      topologyKey:
+        default: kubernetes.io/hostname
+  resources:
+    enabled: false
+    rgw:
+      requests:
+        memory: "5Mi"
+        cpu: "250m"
+      limits:
+        memory: "50Mi"
+        cpu: "500m"
+    jobs:
+      ceph-rgw-storage-init:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      image_repo_sync:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      ks-endpoints:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      ks_service:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+      ks_user:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"
+secrets:
+  keyrings:
+    mon: ceph-mon-keyring
+    mds: ceph-bootstrap-mds-keyring
+    osd: ceph-bootstrap-osd-keyring
+    rgw: os-ceph-bootstrap-rgw-keyring
+    mgr: ceph-bootstrap-mgr-keyring
+    admin: pvc-ceph-client-key
+  identity:
+    admin: ceph-keystone-admin
+    swift: ceph-keystone-user
+    user_rgw: ceph-keystone-user-rgw
+  tls:
+    object_store:
+      api:
+        public: ceph-tls-public
+
+network:
+  api:
+    ingress:
+      public: true
+      classes:
+        namespace: "nginx"
+        cluster: "nginx-cluster"
+      annotations:
+        nginx.ingress.kubernetes.io/rewrite-target: /
+        nginx.ingress.kubernetes.io/proxy-body-size: "0"
+        nginx.org/proxy-max-temp-file-size: "0"
+    external_policy_local: false
+    node_port:
+      enabled: false
+      port: 30004
+  public: 192.168.0.0/16
+  cluster: 192.168.0.0/16
+
+conf:
+  templates:
+    keyring:
+      admin: |
+        [client.admin]
+          key = {{ key }}
+          auid = 0
+          caps mds = "allow"
+          caps mon = "allow *"
+          caps osd = "allow *"
+          caps mgr = "allow *"
+      bootstrap:
+        rgw: |
+          [client.bootstrap-rgw]
+            key = {{ key }}
+            caps mgr = "allow profile bootstrap-rgw"
+  features:
+    rgw: true
+  pool:
+  #NOTE(portdirect): this drives a simple approximation of
+  # https://ceph.com/pgcalc/, the `target.osd` key should be set to match the
+  # expected number of osds in a cluster, and the `target.pg_per_osd` should be
+  # set to match the desired number of placement groups on each OSD.
+    crush:
+      #NOTE(portdirect): to use RBD devices with Ubuntu 16.04's 4.4.x series
+      # kernel this should be set to `hammer`
+      tunables: null
+    target:
+      #NOTE(portdirect): arbitrarily we set the default number of expected OSD's to 5
+      # to match the number of nodes in the OSH gate.
+      osd: 5
+      pg_per_osd: 100
+    default:
+      #NOTE(portdirect): this should be 'same_host' for a single node
+      # cluster to be in a healthy state
+      crush_rule: replicated_rule
+    #NOTE(portdirect): this section describes the pools that will be managed by
+    # the ceph pool management job, as it tunes the pgs and crush rule, based on
+    # the above.
+    spec:
+      # RBD pool
+      - name: rbd
+        application: rbd
+        replication: 3
+        percent_total_data: 40
+      # CephFS pools
+      - name: cephfs_metadata
+        application: cephfs
+        replication: 3
+        percent_total_data: 5
+      - name: cephfs_data
+        application: cephfs
+        replication: 3
+        percent_total_data: 10
+      # RadosGW pools
+      - name: .rgw.root
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.control
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.data.root
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.gc
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.log
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.intent-log
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.meta
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.usage
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.users.keys
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.users.email
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.users.swift
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.users.uid
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.buckets.extra
+        application: rgw
+        replication: 3
+        percent_total_data: 0.1
+      - name: default.rgw.buckets.index
+        application: rgw
+        replication: 3
+        percent_total_data: 3
+      - name: default.rgw.buckets.data
+        application: rgw
+        replication: 3
+        percent_total_data: 34.8
+  rgw_ks:
+    enabled: false
+    config:
+      rgw_keystone_api_version: 3
+      rgw_keystone_accepted_roles: "admin, member"
+      rgw_keystone_implicit_tenants: true
+      rgw_keystone_make_new_tenants: true
+      rgw_s3_auth_use_keystone: true
+      rgw_swift_account_in_url: true
+      rgw_swift_url: null
+      #NOTE (portdirect): See http://tracker.ceph.com/issues/21226
+      rgw_keystone_token_cache_size: 0
+  ceph:
+    global:
+      # auth
+      cephx: true
+      cephx_require_signatures: false
+      cephx_cluster_require_signatures: true
+      cephx_service_require_signatures: false
+    osd:
+      osd_mkfs_type: xfs
+      osd_mkfs_options_xfs: -f -i size=2048
+      osd_max_object_name_len: 256
+      ms_bind_port_min: 6800
+      ms_bind_port_max: 7100
+
+dependencies:
+  dynamic:
+    common:
+      local_image_registry:
+        jobs:
+          - ceph-client-image-repo-sync
+        services:
+          - endpoint: node
+            service: local_image_registry
+  static:
+    bootstrap:
+      jobs: null
+      services:
+        - endpoint: internal
+          service: ceph_mon
+    rgw:
+      jobs:
+        - ceph-rgw-storage-init
+      services:
+        - endpoint: internal
+          service: keystone-api
+    image_repo_sync:
+      services:
+        - endpoint: internal
+          service: local_image_registry
+    ks_endpoints:
+      jobs:
+        - ceph-ks-service
+      services:
+        - endpoint: internal
+          service: identity
+    ks_service:
+      services:
+        - endpoint: internal
+          service: identity
+    ks_user:
+      services:
+        - endpoint: internal
+          service: identity
+bootstrap:
+  enabled: false
+  script: |
+    ceph -s
+    function ensure_pool () {
+      ceph osd pool stats $1 || ceph osd pool create $1 $2
+      local test_luminous=$(ceph tell osd.* version | egrep -c "12.2|luminous")
+      if [[ ${test_luminous} -gt 0 ]]; then
+        ceph osd pool application enable $1 $3
+      fi
+    }
+    #ensure_pool volumes 8 cinder
+
+
+endpoints:
+  cluster_domain_suffix: cluster.local
+  local_image_registry:
+    name: docker-registry
+    namespace: docker-registry
+    hosts:
+      default: localhost
+      internal: docker-registry
+      node: localhost
+    host_fqdn_override:
+      default: null
+    port:
+      registry:
+        node: 5000
+  identity:
+    name: keystone
+    namespace: null
+    auth:
+      admin:
+        region_name: RegionOne
+        username: admin
+        password: password
+        project_name: admin
+        user_domain_name: default
+        project_domain_name: default
+      swift:
+        role: admin
+        region_name: RegionOne
+        username: swift
+        password: password
+        project_name: service
+        user_domain_name: service
+        project_domain_name: service
+    hosts:
+      default: keystone-api
+      public: keystone
+    host_fqdn_override:
+      default: null
+    path:
+      default: /v3
+    scheme:
+      default: http
+    port:
+      api:
+        default: 80
+        internal: 5000
+  object_store:
+    name: swift
+    namespace: null
+    hosts:
+      default: ceph-rgw
+      public: radosgw
+    host_fqdn_override:
+      default: null
+      # NOTE(portdirect): this chart supports TLS for fqdn over-ridden public
+      # endpoints using the following format:
+      # public:
+      #   host: null
+      #   tls:
+      #     crt: null
+      #     key: null
+    path:
+      default: /swift/v1/KEY_$(tenant_id)s
+    scheme:
+      default: http
+    port:
+      api:
+        default: 8088
+        public: 80
+  ceph_mon:
+    namespace: null
+    hosts:
+      default: ceph-mon
+      discovery: ceph-mon-discovery
+    host_fqdn_override:
+      default: null
+    port:
+      mon:
+        default: 6789
+
+
+manifests:
+  configmap_ceph_templates: true
+  configmap_bin: true
+  configmap_bin_ks: true
+  configmap_etc: true
+  deployment_rgw: true
+  ingress_rgw: true
+  job_ceph_rgw_storage_init: true
+  job_image_repo_sync: true
+  job_ks_endpoints: true
+  job_ks_service: true
+  job_ks_user: true
+  secret_keystone_rgw: true
+  secret_ingress_tls: true
+  secret_keystone_rgw: true
+  secret_keystone: true
+  service_ingress_rgw: true
+  service_rgw: true

doc/source/install/developer/deploy-with-ceph.rst

@@ -113,8 +113,8 @@ Alternatively, this step can be performed by running the script directly:
 
   ./tools/deployment/developer/ceph/100-horizon.sh
 
-Create Ceph endpoints and service account for use with keystone
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Deploy Rados Gateway for object store
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. literalinclude:: ../../../../tools/deployment/developer/ceph/110-ceph-radosgateway.sh
     :language: shell

doc/source/install/multinode.rst

@@ -200,8 +200,8 @@ Alternatively, this step can be performed by running the script directly:
 
   ./tools/deployment/multinode/080-keystone.sh
 
-Create Ceph endpoints and service account for use with keystone
----------------------------------------------------------------
+Deploy Rados Gateway for object store
+-------------------------------------
 
 .. literalinclude:: ../../../tools/deployment/multinode/090-ceph-radosgateway.sh
     :language: shell

tools/deployment/armada/multinode/armada-osh.yaml

@@ -357,11 +357,11 @@ data:
           swift:
             password: ${SWIFT_USER_PASSWORD}
           object_store:
-            namespace: ceph
+            namespace: openstack
           ceph_mon:
             namespace: ceph
       object_store:
-        namespace: ceph
+        namespace: openstack
       ceph_mon:
         namespace: ceph
     labels:
@@ -378,7 +378,7 @@ data:
       cluster: ${CEPH_NETWORK}
     deployment:
       storage_secrets: False
-      ceph: False
+      ceph: True
       rbd_provisioner: False
       cephfs_provisioner: False
       client_secrets: False
@@ -386,7 +386,7 @@ data:
   source:
     type: local
     location: ${OSH_PATH}
-    subpath: ceph-client
+    subpath: ceph-rgw
     reference: master
   dependencies:
     - helm-toolkit

tools/deployment/developer/ceph/110-ceph-radosgateway.sh

@@ -26,7 +26,7 @@ endpoints:
   identity:
     namespace: openstack
   object_store:
-    namespace: ceph
+    namespace: openstack
   ceph_mon:
     namespace: ceph
 network:
@@ -34,7 +34,7 @@ network:
   cluster: 172.17.0.1/16
 deployment:
   storage_secrets: false
-  ceph: false
+  ceph: true
   rbd_provisioner: false
   cephfs_provisioner: false
   client_secrets: false
@@ -44,8 +44,11 @@ bootstrap:
 conf:
   rgw_ks:
     enabled: true
+pod:
+  replicas:
+    rgw: 1
 EOF
-helm upgrade --install radosgw-openstack ./ceph-client \
+helm upgrade --install radosgw-openstack ./ceph-rgw \
   --namespace=openstack \
   --values=/tmp/radosgw-openstack.yaml \
   ${OSH_EXTRA_HELM_ARGS} \

tools/deployment/multinode/090-ceph-radosgateway.sh

@@ -24,7 +24,7 @@ endpoints:
   identity:
     namespace: openstack
   object_store:
-    namespace: ceph
+    namespace: openstack
   ceph_mon:
     namespace: ceph
 network:
@@ -32,7 +32,7 @@ network:
   cluster: ${CEPH_CLUSTER_NETWORK}
 deployment:
   storage_secrets: false
-  ceph: false
+  ceph: true
   rbd_provisioner: false
   cephfs_provisioner: false
   client_secrets: false
@@ -43,7 +43,7 @@ conf:
   rgw_ks:
     enabled: true
 EOF
-helm upgrade --install radosgw-openstack ./ceph-client \
+helm upgrade --install radosgw-openstack ./ceph-rgw \
   --namespace=openstack \
   --values=/tmp/radosgw-openstack.yaml \
   ${OSH_EXTRA_HELM_ARGS} \