From e12d7621362e56dbf34391aea11f6825f6645aca Mon Sep 17 00:00:00 2001 From: Chris Wedgwood Date: Fri, 15 Sep 2017 18:55:09 +0000 Subject: [PATCH] [libvirt] Move 'ceph secret' to daemon start up At present (Kubernetes 1.7.x) there is no logging for postStart operations, when things this causes the pod to restart. This change means we once again get logging. Change-Id: Id046968eef404c05a4215d73406a4eb604eb3ee5 --- .../templates/bin/_ceph-secret-define.sh.tpl | 71 ------------------- libvirt/templates/bin/_libvirt.sh.tpl | 57 ++++++++++++++- libvirt/templates/configmap-bin.yaml | 2 - libvirt/templates/daemonset-libvirt.yaml | 18 ----- 4 files changed, 56 insertions(+), 92 deletions(-) delete mode 100644 libvirt/templates/bin/_ceph-secret-define.sh.tpl diff --git a/libvirt/templates/bin/_ceph-secret-define.sh.tpl b/libvirt/templates/bin/_ceph-secret-define.sh.tpl deleted file mode 100644 index b35ef2bd02..0000000000 --- a/libvirt/templates/bin/_ceph-secret-define.sh.tpl +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -set -x -LIBVIRT_SECRET_DEF=$(mktemp --suffix .xml) -function cleanup { - rm -f ${LIBVIRT_SECRET_DEF} -} -trap cleanup EXIT - -set -ex -# Wait for the libvirtd is up -TIMEOUT=60 -while [[ ! -f /var/run/libvirtd.pid ]]; do - if [[ ${TIMEOUT} -gt 0 ]]; then - let TIMEOUT-=1 - sleep 1 - else - echo "ERROR: Libvirt did not start in time (pid file missing)" - exit 1 - fi -done - -# Even though we see the pid file the socket immediately (this is -# needed for virsh) -TIMEOUT=10 -while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do - if [[ ${TIMEOUT} -gt 0 ]]; then - let TIMEOUT-=1 - sleep 1 - else - echo "ERROR: Libvirt did not start in time (socket missing)" - exit 1 - fi -done - -if [ -z "${LIBVIRT_CEPH_SECRET_UUID}" ] ; then - echo "ERROR: No Libvirt Secret UUID Supplied" - exit 1 -fi - -if [ -z "${CEPH_CINDER_KEYRING}" ] ; then - CEPH_CINDER_KEYRING=$(sed -n 's/^[[:space:]]*key[[:blank:]]\+=[[:space:]]\(.*\)/\1/p' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring) -fi - -cat > ${LIBVIRT_SECRET_DEF} < - ${LIBVIRT_CEPH_SECRET_UUID} - - client.${CEPH_CINDER_USER}. secret - - -EOF - -virsh secret-define --file ${LIBVIRT_SECRET_DEF} -virsh secret-set-value --secret "${LIBVIRT_CEPH_SECRET_UUID}" --base64 "${CEPH_CINDER_KEYRING}" diff --git a/libvirt/templates/bin/_libvirt.sh.tpl b/libvirt/templates/bin/_libvirt.sh.tpl index 97cef23079..28dae52109 100644 --- a/libvirt/templates/bin/_libvirt.sh.tpl +++ b/libvirt/templates/bin/_libvirt.sh.tpl @@ -30,4 +30,59 @@ if [[ -c /dev/kvm ]]; then chown root:kvm /dev/kvm fi -exec libvirtd --listen +libvirtd --listen & + +LIBVIRT_SECRET_DEF=$(mktemp --suffix .xml) +function cleanup { + rm -f ${LIBVIRT_SECRET_DEF} +} +trap cleanup EXIT + +# Wait for the libvirtd is up +TIMEOUT=60 +while [[ ! -f /var/run/libvirtd.pid ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (pid file missing)" + exit 1 + fi +done + +# Even though we see the pid file the socket immediately (this is +# needed for virsh) +TIMEOUT=10 +while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (socket missing)" + exit 1 + fi +done + +if [ -z "${LIBVIRT_CEPH_SECRET_UUID}" ] ; then + echo "ERROR: No libvirt Secret UUID Supplied" + exit 1 +fi + +if [ -z "${CEPH_CINDER_KEYRING}" ] ; then + CEPH_CINDER_KEYRING=$(sed -n 's/^[[:space:]]*key[[:blank:]]\+=[[:space:]]\(.*\)/\1/p' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring) +fi + +cat > ${LIBVIRT_SECRET_DEF} < + ${LIBVIRT_CEPH_SECRET_UUID} + + client.${CEPH_CINDER_USER}. secret + + +EOF + +virsh secret-define --file ${LIBVIRT_SECRET_DEF} +virsh secret-set-value --secret "${LIBVIRT_CEPH_SECRET_UUID}" --base64 "${CEPH_CINDER_KEYRING}" + +# rejoin libvirtd +wait diff --git a/libvirt/templates/configmap-bin.yaml b/libvirt/templates/configmap-bin.yaml index 9f7a670fb9..30e00a2f3c 100644 --- a/libvirt/templates/configmap-bin.yaml +++ b/libvirt/templates/configmap-bin.yaml @@ -25,8 +25,6 @@ data: libvirt.sh: | {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.ceph.enabled }} - ceph-secret-define.sh: | -{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} ceph-keyring.sh: |+ {{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/libvirt/templates/daemonset-libvirt.yaml b/libvirt/templates/daemonset-libvirt.yaml index 653eaa2fed..c26c65b008 100644 --- a/libvirt/templates/daemonset-libvirt.yaml +++ b/libvirt/templates/daemonset-libvirt.yaml @@ -87,22 +87,8 @@ spec: - name: LIBVIRT_CEPH_SECRET_UUID value: "{{ .Values.ceph.secret_uuid }}" {{ end }} - {{- if .Values.ceph.enabled }} - lifecycle: - postStart: - exec: - command: - - /tmp/ceph-secret-define.sh - {{- end }} command: - /tmp/libvirt.sh - {{- if .Values.ceph.enabled }} - lifecycle: - postStart: - exec: - command: - - /tmp/ceph-secret-define.sh - {{- end }} volumeMounts: - name: libvirt-bin mountPath: /tmp/libvirt.sh @@ -143,10 +129,6 @@ spec: mountPath: /tmp/client-keyring subPath: key readOnly: true - - name: libvirt-bin - mountPath: /tmp/ceph-secret-define.sh - subPath: ceph-secret-define.sh - readOnly: true {{- end }} {{ if $mounts_libvirt.volumeMounts }}{{ toYaml $mounts_libvirt.volumeMounts | indent 12 }}{{ end }} volumes: