Merge "Ceph/Libvirt/Cinder/Nova: update charts to support OpenStack >= Ocata"

cinder/values.yaml

@@ -748,6 +748,7 @@ conf:
       rbd_store_chunk_size: 4
       rados_connect_timeout: -1
       rbd_user: cinder
+      rbd_secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
   rally_tests:
     run_tempest: false
     tests:

glance/templates/bin/_ceph-admin-keyring.sh.tpl

@@ -19,7 +19,7 @@ limitations under the License.
 set -ex
 export HOME=/tmp
 
-cat <<EOF > /etc/ceph/ceph.client.admin.keyring
+cat > /etc/ceph/ceph.client.admin.keyring << EOF
 [client.admin]
 {{- if .Values.conf.ceph.admin_keyring }}
     key = {{ .Values.conf.ceph.admin_keyring }}

glance/templates/bin/_ceph-keyring.sh.tpl

@@ -19,7 +19,7 @@ limitations under the License.
 set -ex
 export HOME=/tmp
 
-cat <<EOF > /etc/ceph/ceph.client.${RBD_STORE_USER}.keyring
+cat > /etc/ceph/ceph.client.${RBD_STORE_USER}.keyring <<EOF
 [client.${RBD_STORE_USER}]
     key = $(cat /tmp/client-keyring)
 EOF

libvirt/templates/bin/_ceph-admin-keyring.sh.tpl

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+export HOME=/tmp
+
+cat > /etc/ceph/ceph.client.admin.keyring << EOF
+[client.admin]
+{{- if .Values.conf.ceph.admin_keyring }}
+    key = {{ .Values.conf.ceph.admin_keyring }}
+{{- else }}
+    key = $(cat /tmp/client-keyring)
+{{- end }}
+EOF
+
+exit 0

libvirt/templates/bin/_ceph-keyring.sh.tpl

@@ -19,13 +19,21 @@ limitations under the License.
 set -ex
 export HOME=/tmp
 
-cat <<EOF > /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring
+KEYRING=/etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring
-[client.{{ .Values.ceph.cinder_user }}]
+{{- if .Values.conf.ceph.cinder.keyring }}
-{{- if .Values.ceph.cinder_keyring }}
+cat > ${KEYRING} <<EOF
-    key = {{ .Values.ceph.cinder_keyring }}
+[client.{{ .Values.conf.ceph.cinder.user }}]
-{{- else }}
+    key = {{ .Values.conf.ceph.cinder.keyring }}
-    key = $(cat /tmp/client-keyring)
-{{- end }}
 EOF
+{{- else }}
+if ! [ "x${CEPH_CINDER_USER}" == "xadmin"]; then
+  #NOTE(Portdirect): Determine proper privs to assign keyring
+  ceph auth get-or-create client.${CEPH_CINDER_USER} \
+    mon "allow *" \
+    osd "allow *" \
+    mgr "allow *" \
+    -o ${KEYRING}
 
-exit 0
+  rm -f /etc/ceph/ceph.client.admin.keyring
+fi
+{{- end }}

libvirt/templates/bin/_libvirt.sh.tpl

@@ -38,7 +38,7 @@ if [ -d /sys/kernel/mm/hugepages ]; then
   fi
 fi
 
-if [ -n "${LIBVIRT_CEPH_SECRET_UUID}" ] ; then
+if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] ; then
   libvirtd --listen &
 
   tmpsecret=$(mktemp --suffix .xml)
@@ -78,7 +78,7 @@ if [ -n "${LIBVIRT_CEPH_SECRET_UUID}" ] ; then
 
   cat > ${tmpsecret} <<EOF
 <secret ephemeral='no' private='no'>
-  <uuid>${LIBVIRT_CEPH_SECRET_UUID}</uuid>
+  <uuid>${LIBVIRT_CEPH_CINDER_SECRET_UUID}</uuid>
   <usage type='ceph'>
     <name>client.${CEPH_CINDER_USER}. secret</name>
   </usage>
@@ -86,7 +86,7 @@ if [ -n "${LIBVIRT_CEPH_SECRET_UUID}" ] ; then
 EOF
 
   virsh secret-define --file ${tmpsecret}
-  virsh secret-set-value --secret "${LIBVIRT_CEPH_SECRET_UUID}" --base64 "${CEPH_CINDER_KEYRING}"
+  virsh secret-set-value --secret "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" --base64 "${CEPH_CINDER_KEYRING}"
 
   # rejoin libvirtd
   wait

libvirt/templates/configmap-bin.yaml

@@ -24,8 +24,10 @@ metadata:
 data:
   libvirt.sh: |
 {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- if .Values.ceph.enabled }}
+{{- if .Values.conf.ceph.enabled }}
   ceph-keyring.sh: |
 {{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-admin-keyring.sh: |
+{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
 {{- end }}

libvirt/templates/daemonset-libvirt.yaml

@@ -44,9 +44,31 @@ spec:
       hostNetwork: true
       hostPID: true
       dnsPolicy: ClusterFirstWithHostNet
-{{- if .Values.ceph.enabled }}
       initContainers:
 {{ tuple $envAll $dependencies $mounts_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+{{- if .Values.conf.ceph.enabled }}
+        {{- if empty .Values.conf.ceph.cinder.keyring }}
+        - name: ceph-admin-keyring-placement
+          image: {{ .Values.images.tags.libvirt }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-admin-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: libvirt-bin
+              mountPath: /tmp/ceph-admin-keyring.sh
+              subPath: ceph-admin-keyring.sh
+              readOnly: true
+            {{- if empty .Values.conf.ceph.admin_keyring }}
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+            {{ end }}
+        {{ end }}
         - name: ceph-keyring-placement
           image: {{ .Values.images.tags.libvirt }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -54,13 +76,13 @@ spec:
             runAsUser: 0
           env:
             - name: CEPH_CINDER_USER
-              value: "{{ .Values.ceph.cinder_user }}"
+              value: "{{ .Values.conf.ceph.cinder.user }}"
-            {{- if .Values.ceph.cinder_keyring }}
+            {{- if .Values.conf.ceph.cinder.keyring }}
             - name: CEPH_CINDER_KEYRING
-              value: "{{ .Values.ceph.cinder_keyring }}"
+              value: "{{ .Values.conf.ceph.cinder.keyring }}"
             {{ end }}
-            - name: LIBVIRT_CEPH_SECRET_UUID
+            - name: LIBVIRT_CEPH_CINDER_SECRET_UUID
-              value: "{{ .Values.ceph.secret_uuid }}"
+              value: "{{ .Values.conf.ceph.cinder.secret_uuid }}"
           command:
             - /tmp/ceph-keyring.sh
           volumeMounts:
@@ -70,12 +92,10 @@ spec:
               mountPath: /tmp/ceph-keyring.sh
               subPath: ceph-keyring.sh
               readOnly: true
-            {{- if empty .Values.ceph.cinder_keyring }}
+            - name: ceph-etc
-            - name: ceph-keyring
+              mountPath: /etc/ceph/ceph.conf
-              mountPath: /tmp/client-keyring
+              subPath: ceph.conf
-              subPath: key
               readOnly: true
-            {{ end }}
 {{- end }}
       containers:
         - name: libvirt
@@ -85,16 +105,16 @@ spec:
           securityContext:
             privileged: true
             runAsUser: 0
-          {{- if .Values.ceph.enabled }}
+          {{- if .Values.conf.ceph.enabled }}
           env:
             - name: CEPH_CINDER_USER
-              value: "{{ .Values.ceph.cinder_user }}"
+              value: "{{ .Values.conf.ceph.cinder.user }}"
-            {{- if .Values.ceph.cinder_keyring }}
+            {{- if .Values.conf.ceph.cinder.keyring }}
             - name: CEPH_CINDER_KEYRING
-              value: "{{ .Values.ceph.cinder_keyring }}"
+              value: "{{ .Values.conf.ceph.cinder.keyring }}"
             {{ end }}
-            - name: LIBVIRT_CEPH_SECRET_UUID
+            - name: LIBVIRT_CEPH_CINDER_SECRET_UUID
-              value: "{{ .Values.ceph.secret_uuid }}"
+              value: "{{ .Values.conf.ceph.cinder.secret_uuid }}"
           {{ end }}
           command:
             - /tmp/libvirt.sh
@@ -129,14 +149,14 @@ spec:
             - name: machine-id
               mountPath: /etc/machine-id
               readOnly: true
-            {{- if .Values.ceph.enabled }}
+            {{- if .Values.conf.ceph.enabled }}
             - name: etcceph
               mountPath: /etc/ceph
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            {{- if empty .Values.ceph.cinder_keyring }}
+            {{- if empty .Values.conf.ceph.cinder.keyring }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
@@ -153,14 +173,14 @@ spec:
           configMap:
             name: libvirt-etc
             defaultMode: 0444
-        {{- if .Values.ceph.enabled }}
+        {{- if .Values.conf.ceph.enabled }}
         - name: etcceph
           emptyDir: {}
         - name: ceph-etc
           configMap:
             name: ceph-etc
             defaultMode: 0444
-        {{- if empty .Values.ceph.cinder_keyring }}
+        {{- if empty .Values.conf.ceph.cinder.keyring }}
         - name: ceph-keyring
           secret:
             secretName: pvc-ceph-client-key

libvirt/values.yaml

@@ -31,14 +31,15 @@ images:
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
   pull_policy: "IfNotPresent"
 
-ceph:
-  enabled: true
-  monitors: []
-  cinder_user: "admin"
-  cinder_keyring: null
-  secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
 
 conf:
+  ceph:
+    enabled: true
+    admin_keyring: null
+    cinder:
+      user: "cinder"
+      keyring: null
+      secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
   libvirt:
     listen_tcp: "1"
     listen_tls: "0"

nova/templates/bin/_ceph-admin-keyring.sh.tpl

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ex
+export HOME=/tmp
+
+cat > /etc/ceph/ceph.client.admin.keyring << EOF
+[client.admin]
+{{- if .Values.conf.ceph.admin_keyring }}
+    key = {{ .Values.conf.ceph.admin_keyring }}
+{{- else }}
+    key = $(cat /tmp/client-keyring)
+{{- end }}
+EOF
+
+exit 0

nova/templates/bin/_ceph-keyring.sh.tpl

@@ -19,10 +19,21 @@ limitations under the License.
 set -ex
 export HOME=/tmp
 
-CEPH_CINDER_KEYRING_FILE="/etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring"
+KEYRING=/etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring
-echo "[client.${CEPH_CINDER_USER}]" > ${CEPH_CINDER_KEYRING_FILE}
+{{- if .Values.conf.ceph.cinder.keyring }}
-if ! [ -z "${CEPH_CINDER_KEYRING}" ] ; then
+cat > ${KEYRING} <<EOF
-  echo "    key = ${CEPH_CINDER_KEYRING}" >> ${CEPH_CINDER_KEYRING_FILE}
+[client.{{ .Values.conf.ceph.cinder.user }}]
-else
+    key = {{ .Values.conf.ceph.cinder.keyring }}
-  echo "    key = $(cat /tmp/client-keyring)" >> ${CEPH_CINDER_KEYRING_FILE}
+EOF
+{{- else }}
+if ! [ "x${CEPH_CINDER_USER}" == "xadmin"]; then
+  #NOTE(Portdirect): Determine proper privs to assign keyring
+  ceph auth get-or-create client.${CEPH_CINDER_USER} \
+    mon "allow *" \
+    osd "allow *" \
+    mgr "allow *" \
+    -o ${KEYRING}
+
+  rm -f /etc/ceph/ceph.client.admin.keyring
 fi
+{{- end }}

nova/templates/bin/_nova-compute.sh.tpl

@@ -18,16 +18,6 @@ limitations under the License.
 
 set -ex
 
-console_kind="{{- .Values.console.console_kind -}}"
-if [ "${console_kind}" == "novnc" ] ; then
 exec nova-compute \
       --config-file /etc/nova/nova.conf \
-      --config-file /tmp/pod-shared/nova-vnc.ini
+      --config-file /tmp/pod-shared/nova-console.conf
-elif [ "${console_kind}" == "spice" ] ; then
-exec nova-compute \
-      --config-file /etc/nova/nova.conf \
-      --config-file /tmp/pod-shared/nova-spice.ini
-else
-exec nova-compute \
-      --config-file /etc/nova/nova.conf
-fi

nova/templates/bin/_nova-console-compute-init.sh.tpl

@@ -46,14 +46,15 @@ if [ -z "${listen_ip}" ] ; then
     listen_ip=0.0.0.0
 fi
 
+touch /tmp/pod-shared/nova-console.conf
 if [ "${console_kind}" == "novnc" ] ; then
-cat <<EOF>/tmp/pod-shared/nova-vnc.ini
+  cat > /tmp/pod-shared/nova-console.conf <<EOF
 [vnc]
 vncserver_proxyclient_address = $client_address
 vncserver_listen = $listen_ip
 EOF
 elif [ "${console_kind}" == "spice" ] ; then
-cat <<EOF>/tmp/pod-shared/nova-spice.ini
+  cat > /tmp/pod-shared/nova-console.conf <<EOF
 [spice]
 server_proxyclient_address = $client_address
 server_listen = $listen_ip

nova/templates/configmap-bin.yaml

@@ -41,9 +41,11 @@ data:
 {{- include "helm-toolkit.scripts.keystone_endpoints" . | indent 4 }}
   ks-user.sh: |
 {{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
-{{- if .Values.ceph.enabled }}
+{{- if .Values.conf.ceph.enabled }}
   ceph-keyring.sh: |
 {{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-admin-keyring.sh: |
+{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
   nova-api.sh: |
 {{ tuple "bin/_nova-api.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

nova/templates/daemonset-compute.yaml

@@ -64,7 +64,29 @@ spec:
               readOnly: true
             - name: varlibnova
               mountPath: /var/lib/nova
-        {{- if .Values.ceph.enabled }}
+        {{- if .Values.conf.ceph.enabled }}
+        {{- if empty .Values.conf.ceph.cinder.keyring }}
+        - name: ceph-admin-keyring-placement
+          image: {{ .Values.images.tags.nova_compute }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          command:
+            - /tmp/ceph-admin-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: nova-bin
+              mountPath: /tmp/ceph-admin-keyring.sh
+              subPath: ceph-admin-keyring.sh
+              readOnly: true
+            {{- if empty .Values.conf.ceph.admin_keyring }}
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+            {{ end }}
+        {{ end }}
         - name: ceph-keyring-placement
           image: {{ .Values.images.tags.nova_compute }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
@@ -72,13 +94,13 @@ spec:
             runAsUser: {{ .Values.pod.user.nova.uid }}
           env:
             - name: CEPH_CINDER_USER
-              value: "{{ .Values.ceph.cinder_user }}"
+              value: "{{ .Values.conf.ceph.cinder.user }}"
-            {{- if .Values.ceph.cinder_keyring }}
+            {{- if .Values.conf.ceph.cinder.keyring }}
             - name: CEPH_CINDER_KEYRING
-              value: "{{ .Values.ceph.cinder_keyring }}"
+              value: "{{ .Values.conf.ceph.cinder.keyring }}"
             {{ end }}
             - name: LIBVIRT_CEPH_SECRET_UUID
-              value: "{{ .Values.ceph.secret_uuid }}"
+              value: "{{ .Values.conf.ceph.secret_uuid }}"
           command:
             - /tmp/ceph-keyring.sh
           volumeMounts:
@@ -87,13 +109,11 @@ spec:
             - name: nova-bin
               mountPath: /tmp/ceph-keyring.sh
               subPath: ceph-keyring.sh
-            {{- if empty .Values.ceph.cinder_keyring }}
+            - name: ceph-etc
-            - name: ceph-keyring
+              mountPath: /etc/ceph/ceph.conf
-              mountPath: /tmp/client-keyring
+              subPath: ceph.conf
-              subPath: key
               readOnly: true
         {{ end }}
-        {{ end }}
         {{- if eq .Values.console.console_kind "novnc"}}
         - name: nova-compute-vnc-init
           image: {{ .Values.images.tags.nova_compute }}
@@ -136,16 +156,16 @@ spec:
           securityContext:
             runAsUser: {{ .Values.pod.user.nova.uid }}
             privileged: true
-          {{- if .Values.ceph.enabled }}
+          {{- if .Values.conf.ceph.enabled }}
           env:
             - name: CEPH_CINDER_USER
-              value: "{{ .Values.ceph.cinder_user }}"
+              value: "{{ .Values.conf.ceph.cinder.user }}"
-            {{- if .Values.ceph.cinder_keyring }}
+            {{- if .Values.conf.ceph.cinder.keyring }}
             - name: CEPH_CINDER_KEYRING
-              value: "{{ .Values.ceph.cinder_keyring }}"
+              value: "{{ .Values.conf.ceph.cinder.keyring }}"
             {{ end }}
             - name: LIBVIRT_CEPH_SECRET_UUID
-              value: "{{ .Values.ceph.secret_uuid }}"
+              value: "{{ .Values.conf.ceph.secret_uuid }}"
           {{ end }}
           command:
             - /tmp/nova-compute.sh
@@ -193,14 +213,14 @@ spec:
               mountPath: /root/.ssh/config
               subPath: ssh-config
               readOnly: true
-            {{- if .Values.ceph.enabled }}
+            {{- if .Values.conf.ceph.enabled }}
             - name: etcceph
               mountPath: /etc/ceph
             - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            {{- if empty .Values.ceph.cinder_keyring }}
+            {{- if empty .Values.conf.ceph.cinder.keyring }}
             - name: ceph-keyring
               mountPath: /tmp/client-keyring
               subPath: key
@@ -266,14 +286,14 @@ spec:
           configMap:
             name: {{ $configMapName }}
             defaultMode: 0444
-        {{- if .Values.ceph.enabled }}
+        {{- if .Values.conf.ceph.enabled }}
         - name: etcceph
           emptyDir: {}
         - name: ceph-etc
           configMap:
             name: ceph-etc
             defaultMode: 0444
-        {{- if empty .Values.ceph.cinder_keyring }}
+        {{- if empty .Values.conf.ceph.cinder.keyring }}
         - name: ceph-keyring
           secret:
             secretName: pvc-ceph-client-key

nova/values.yaml

@@ -196,13 +196,6 @@ network:
     name: "nova-ssh"
     port: 8022
 
-ceph:
-  enabled: true
-  monitors: []
-  cinder_user: "admin"
-  cinder_keyring: null
-  secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
-
 dependencies:
   dynamic:
     targeted:
@@ -413,6 +406,13 @@ ssh:
     - ed25519
 
 conf:
+  ceph:
+    enabled: true
+    admin_keyring: null
+    cinder:
+      user: "cinder"
+      keyring: null
+      secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
   ssh:
     override:
     append:
@@ -1070,7 +1070,7 @@ conf:
       images_type: qcow2
       images_rbd_pool: vms
       images_rbd_ceph_conf: /etc/ceph/ceph.conf
-      rbd_user: admin
+      rbd_user: cinder
       rbd_secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337
       disk_cachemodes: "network=writeback"
       hw_disk_discard: unmap

tools/deployment/developer/nfs/150-libvirt.sh

@@ -22,7 +22,7 @@ make libvirt
 : ${OSH_EXTRA_HELM_ARGS:=""}
 helm upgrade --install libvirt ./libvirt \
   --namespace=openstack \
-  --set ceph.enabled=false \
+  --set conf.ceph.enabled=false \
   ${OSH_EXTRA_HELM_ARGS} \
   ${OSH_EXTRA_HELM_ARGS_LIBVIRT}
 

tools/deployment/developer/nfs/160-compute-kit.sh

@@ -25,14 +25,14 @@ if [ "x$(systemd-detect-virt)" == "xnone" ]; then
   echo 'OSH is not being deployed in virtualized environment'
   helm upgrade --install nova ./nova \
       --namespace=openstack \
-      --set ceph.enabled=false \
+      --set conf.ceph.enabled=false \
       ${OSH_EXTRA_HELM_ARGS} \
       ${OSH_EXTRA_HELM_ARGS_NOVA}
 else
   echo 'OSH is being deployed in virtualized environment, using qemu for nova'
   helm upgrade --install nova ./nova \
       --namespace=openstack \
-      --set ceph.enabled=false \
+      --set conf.ceph.enabled=false \
       --set conf.nova.libvirt.virt_type=qemu \
       ${OSH_EXTRA_HELM_ARGS} \
       ${OSH_EXTRA_HELM_ARGS_NOVA}