From a4078d8a7a442c364d470f4dbee14d29a540f789 Mon Sep 17 00:00:00 2001 From: Matthew Heler Date: Wed, 23 Jan 2019 11:06:34 -0600 Subject: [PATCH] [CEPH] Add a support script for manipulating a PV backed by RBD Features of the script include: - creating and removing a snapshot within Ceph for a PV - backing up and restoring a PV within Ceph - listing the RBD image details (watchers, thin/thick provisioned usage, image properties) and snapshot information Take note that all backups are by default stored locally in "/var/lib/openstack-helm/ceph/backup" on the POD hosting this utility container Change-Id: I5de30d5337754a411b5e1162a215596afb469bac --- Dockerfiles/ceph-utility/Dockerfile.ubuntu | 30 ++--- ceph-utility/templates/bin/utility/_nccli.tpl | 2 +- .../templates/bin/utility/_rbd_pv.tpl | 124 ++++++++++++++++++ ceph-utility/templates/configmap-bin.yaml | 3 + .../templates/deployment-utility.yaml | 40 ++++++ ceph-utility/values.yaml | 7 +- docs/rbd_pv.md | 58 ++++++++ 7 files changed, 246 insertions(+), 18 deletions(-) create mode 100644 ceph-utility/templates/bin/utility/_rbd_pv.tpl create mode 100644 docs/rbd_pv.md diff --git a/Dockerfiles/ceph-utility/Dockerfile.ubuntu b/Dockerfiles/ceph-utility/Dockerfile.ubuntu index dcbd9db4..3c2c609f 100755 --- a/Dockerfiles/ceph-utility/Dockerfile.ubuntu +++ b/Dockerfiles/ceph-utility/Dockerfile.ubuntu @@ -1,26 +1,24 @@ -ARG UBUNTU_RELEASE=xenial - -FROM docker.io/ubuntu:${UBUNTU_RELEASE} -LABEL maintainer="sreejith.punnapuzha@outlook.com" +ARG FROM=docker.io/ubuntu:xenial +FROM ${FROM} ARG CEPH_RELEASE=mimic -ARG UBUNTU_RELEASE=xenial +ARG KUBE_VERSION=1.12.2 RUN set -xe \ - && echo '#!/bin/sh' > /usr/sbin/policy-rc.d \ - && echo 'exit 101' >> /usr/sbin/policy-rc.d \ - && chmod +x /usr/sbin/policy-rc.d \ + && export DEBIAN_FRONTEND=noninteractive \ && sed -i '/nobody/d' /etc/passwd \ && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ - && dpkg-divert --local --rename --add /sbin/initctl \ - && cp -a /usr/sbin/policy-rc.d /sbin/initctl \ - && sed -i 's/^exit.*/exit 0/' /sbin/initctl \ - && apt-get update && apt-get install -y wget curl apt-transport-https gnupg\ + && apt-get update && apt-get dist-upgrade -y \ + && apt-get install -y wget curl apt-transport-https ca-certificates gnupg\ && wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - \ - && echo deb https://download.ceph.com/debian-${CEPH_RELEASE}/ ${UBUNTU_RELEASE} main | tee /etc/apt/sources.list.d/ceph.list \ - && apt-get update && apt-get install -y bash python-oslo.rootwrap moreutils vim sudo screen ceph-common python-rbd radosgw rsyslog x11-apps jq \ + && echo deb https://download.ceph.com/debian-${CEPH_RELEASE}/ xenial main | tee /etc/apt/sources.list.d/ceph.list \ + && apt-get update && apt-get install -y bash python-oslo.rootwrap moreutils vim sudo screen ceph ceph-common python-rbd radosgw rsyslog hexedit jq s3cmd rsync xz-utils iperf \ && apt-get remove --purge -y wget apt-transport-https && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && TMP_DIR=$(mktemp --directory) \ + && cd ${TMP_DIR} \ + && curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ + && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \ + && chmod +x /usr/bin/kubectl \ && curl -sSL https://bootstrap.pypa.io/get-pip.py | python \ - && pip --no-cache-dir install --upgrade crush - + && rm -rf ${TMP_DIR} CMD ["/bin/bash"] diff --git a/ceph-utility/templates/bin/utility/_nccli.tpl b/ceph-utility/templates/bin/utility/_nccli.tpl index aef0ccff..810114ea 100644 --- a/ceph-utility/templates/bin/utility/_nccli.tpl +++ b/ceph-utility/templates/bin/utility/_nccli.tpl @@ -14,4 +14,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} -script -f -a -q /var/log/syslog -c "sudo /usr/local/bin/ceph-utility-rootwrap /etc/ceph/rootwrap.conf $*" +script -f -a -q /var/log/syslog -c "sudo -E /usr/local/bin/ceph-utility-rootwrap /etc/ceph/rootwrap.conf $*" diff --git a/ceph-utility/templates/bin/utility/_rbd_pv.tpl b/ceph-utility/templates/bin/utility/_rbd_pv.tpl new file mode 100644 index 00000000..af29303b --- /dev/null +++ b/ceph-utility/templates/bin/utility/_rbd_pv.tpl @@ -0,0 +1,124 @@ +#!/bin/bash + +{{/* +Copyright 2019 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +usage() { + echo "Backup Usage: nccli rbd_pv [-b ] [-n ] [-d (optional, default: /backup)] [-p (optional, default: rbd)]" + echo "Restore Usage: nccli rbd_pv [-r ] [-p (optional, default: rbd)]" + echo "Snapshot Usage: nccli rbd_pv [-b ] [-n ] [-p (optional, default: rbd] [-s (required) ]" + exit 1 +} + +while getopts ":b:p:n:d:r:h:s:" opt; do + case $opt + in + b) pvc_name=${OPTARG};; + n) nspace=${OPTARG};; + d) backup_dest=${OPTARG};; + r) restore_file=${OPTARG};; + p) rbd_pool=${OPTARG};; + s) snapshot=${OPTARG};; + h) usage ;; + esac +done + +if [[ -z "${pvc_name}" || -z "${nspace}" ]]; then + if [[ -z "${restore_file}" ]]; then + usage + echo "ERROR: Missing command line arguement(s)!" + exit 1 + fi +fi + +if [[ -z "${rbd_pool}" ]]; then + rbd_pool="rbd" +fi + +timestamp="$(date +%F_%T)" + +if [[ ! -z "${restore_file}" ]]; then + if [[ -e "${restore_file}" ]]; then + rbd_image="$(echo "${restore_file}" | rev | awk -v FS='/' '{print $1}' | rev | cut -f 1 -d '.')" + if (nccli rbd info "${rbd_pool}"/"${rbd_image}" | grep -q id); then + nccli rbd mv ${rbd_pool}/${rbd_image} ${rbd_pool}/${rbd_image}.orig-${timestamp} + echo "WARNING: Existing PVC/RBD image has been moved to ${rbd_pool}/${rbd_image}.orig-${timestamp}" + fi + nccli rbd import ${restore_file} ${rbd_pool}/${rbd_image} + echo "INFO: Backup has been restored into ${rbd_pool}/${rbd_image}" + else + echo "ERROR: Missing restore file!" + exit 1 + fi +elif [[ ! -z "${snapshot}" ]]; then + volume="$(kubectl -n ${nspace} get pvc ${pvc_name} --no-headers | awk '{ print $3 }')" + rbd_image="$(kubectl get pv "${volume}" -o json | jq -r '.spec.rbd.image')" + + if [[ "x${snapshot}x" == "xcreatex" ]]; then + snap_name="${pvc_name}-${timestamp}" + nccli rbd snap create ${rbd_pool}/${rbd_image}@${snap_name} + echo "INFO: Snapshot ${rbd_pool}/${rbd_image}@${snap_name} has been created for PVC ${pvc_name}" + elif [[ "x${snapshot}x" == "xrollback" ]]; then + snap_name=$(nccli rbd snap ls ${rbd_pool}/${rbd_image}) + nccli rbd snap rollback ${rbd_pool}/${rbd_image}@${snap_name} + echo "WARNING: Rolled back snapshot ${rbd_pool}/${rbd_image}@${snap_name} for ${pvc_name}" + elif [[ "x${snapshot}x" == "xremovex" ]]; then + nccli rbd snap purge ${rbd_pool}/${rbd_image} + echo "Removed snapshot(s) for ${pvc_name}" + elif [[ "x${snapshot}x" == "xshowx" ]]; then + echo "INFO: This PV is mapped to the following RBD Image:" + echo "${rbd_pool}/${rbd_image}" + echo -e "\nINFO: Current open sessions to RBD Image:" + nccli rbd status ${rbd_pool}/${rbd_image} + echo -e "\nINFO: RBD Image information:" + nccli rbd info ${rbd_pool}/${rbd_image} + echo -e "\nINFO: RBD Image snapshot details:" + rbd snap ls ${rbd_pool}/${rbd_image} + echo -e "\nINFO: RBD Image size details:" + nccli rbd du ${rbd_pool}/${rbd_image} + else + echo "ERROR: Missing arguement for snapshot option!" + fi +else + if [[ -z "${backup_dest}" ]]; then + backup_dest="/backup" + fi + if [[ ! -d "${backup_dest}" ]]; then + echo "ERROR: Backup destination does not exist, cannot continue with the backup!" + exit 1 + fi + + echo "INFO: Backing up ${pvc_name} within namespace ${nspace}" + volume="$(kubectl -n ${nspace} get pvc ${pvc_name} --no-headers | awk '{ print $3 }')" + rbd_image="$(kubectl get pv "${volume}" -o json | jq -r '.spec.rbd.image')" + + if [[ -z "${volume}" ]] || (! nccli rbd info "${rbd_pool}"/"${rbd_image}" | grep -q id); then + echo "ERROR: PVC does not exist or is missing! Cannot continue with backup for ${pvc_name}" + exit 1 + else + # Create current snapshot and export to a file + snap_name="${pvc_name}-${timestamp}" + backup_name="${rbd_image}.${pvc_name}-${timestamp}" + nccli rbd snap create ${rbd_pool}/${rbd_image}@${snap_name} + nccli rbd export ${rbd_pool}/${rbd_image}@${snap_name} ${backup_dest}/${backup_name} + # Remove snapshot otherwise we may see an issue cleaning up the PVC from K8s, and from Ceph. + nccli rbd snap rm ${rbd_pool}/${rbd_image}@${snap_name} + echo "INFO: PV ${pvc_name} saved to:" + echo "${backup_dest}/${backup_name}" + fi +fi + +exit 0 diff --git a/ceph-utility/templates/configmap-bin.yaml b/ceph-utility/templates/configmap-bin.yaml index c3bcd42c..c5a202fb 100644 --- a/ceph-utility/templates/configmap-bin.yaml +++ b/ceph-utility/templates/configmap-bin.yaml @@ -44,4 +44,7 @@ data: osd-maintenance: | {{ tuple "bin/utility/_osd-maintenance.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rbd_pv: | +{{ tuple "bin/utility/_rbd_pv.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + {{- end }} diff --git a/ceph-utility/templates/deployment-utility.yaml b/ceph-utility/templates/deployment-utility.yaml index 13bf9452..880ec9d6 100644 --- a/ceph-utility/templates/deployment-utility.yaml +++ b/ceph-utility/templates/deployment-utility.yaml @@ -20,6 +20,35 @@ limitations under the License. {{- $serviceAccountName := printf "%s" $envAll.Release.Name }} {{ tuple $envAll "utility" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - persistentvolumeclaims + - persistentvolumes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- kind: Deployment apiVersion: apps/v1 metadata: @@ -71,6 +100,10 @@ spec: mountPath: /tmp/osd-maintenance subPath: osd-maintenance readOnly: true + - name: ceph-utility-bin + mountPath: /tmp/rbd_pv + subPath: rbd_pv + readOnly: true - name: ceph-utility-sudoers mountPath: /etc/sudoers.d/nccli-sudo subPath: nccli-sudo @@ -90,6 +123,9 @@ spec: mountPath: /etc/ceph/rootwrap.conf subPath: rootwrap.conf readOnly: true + - name: pod-ceph-backup + mountPath: /backup + readOnly: false volumes: - name: ceph-utility-sudoers @@ -112,4 +148,8 @@ spec: secret: secretName: {{ .Values.secrets.keyrings.admin | quote }} defaultMode: 0600 + - name: pod-ceph-backup + hostPath: + path: {{ .Values.conf.storage.utility.backup_target }} + type: DirectoryOrCreate {{- end }} diff --git a/ceph-utility/values.yaml b/ceph-utility/values.yaml index 7b2999de..5c187662 100644 --- a/ceph-utility/values.yaml +++ b/ceph-utility/values.yaml @@ -25,7 +25,7 @@ release_group: null images: pull_policy: IfNotPresent tags: - ceph_utility: 'docker.io/sreejithpunnapuzha/ceph-utility:v0.0.2' + ceph_utility: 'docker.io/sreejithpunnapuzha/ceph-utility:v0.0.3' image_repo_sync: docker.io/docker:17.07.0 local_registry: active: false @@ -92,6 +92,8 @@ conf: radosgw-admin: CommandFilter, radosgw-admin, root rbd: CommandFilter, rbd, root osd-maintenance: CommandFilter, osd-maintenance, root + rbd_pv: CommandFilter, rbd_pv, root + kubectl: CommandFilter, kubectl, root # Below are examples of RegExpFilter. This will restict access to ceph cluster even with admin user #rbd00: RegExpFilter, rbd, root, rbd, (^((?!clone|copy|cp|create|export|export-diff|flatten|import|import-diff|map|merge-diff|pool|remove|rm|rename|mv|resize|unmap).)*$) #rbd01: RegExpFilter, rbd, root, rbd, image-meta, (^((?!get|remove|set).)*$) @@ -126,6 +128,9 @@ conf: # INFO means log all usage # ERROR means only log unsuccessful attempts syslog_log_level: INFO + storage: + utility: + backup_target: /var/lib/openstack-helm/ceph/backup dependencies: dynamic: diff --git a/docs/rbd_pv.md b/docs/rbd_pv.md new file mode 100644 index 00000000..c6057978 --- /dev/null +++ b/docs/rbd_pv.md @@ -0,0 +1,58 @@ +# RBD PVC/PV script + +This MOP covers Maintenance Activities related to using the rbd_pv script +to backup and recover PVCs within your kubernetes environment using Ceph. + +## Usage +Execute nccli rbd_pv without arguements to list usage options. + +``` +nccli rbd_pv +Backup Usage: nccli rbd_pv [-b ] [-n ] [-d (optional, default: /tmp/backup)] [-p (optional, default: rbd)] +Restore Usage: nccli rbd_pv [-r ] [-p (optional, default: rbd)] +Snapshot Usage: nccli rbd_pv [-b ] [-n ] [-p (optional, default: rbd] [-s (required)] +``` + +## Backing up a PVC/PV from RBD +To backup a PV, execute the following: + +``` +nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack +``` + +## Restoring a PVC/PV backup +To restore a PV RBD backup image, execute the following: + +``` +nccli rbd_pv -r /backup/kubernetes-dynamic-pvc-ab1f2e8f-21a4-11e9-ab61-ca77944df03c.img +``` +NOTE: The original PVC/PV will be renamed and not overwritten. +NOTE: Before restoring, you _must_ ensure it is not mounted! + +## Creating a Snapshot for a PVC/PV + +``` +nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s create +``` + +## Rolling back to a Snapshot for a PVC/PV + +``` +nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s rollback +``` + +NOTE: Before rolling back a snapshot, you _must_ ensure the PVC/PV volume is not mounted!! + +## Removing a Snapshot for a PVC/PV + +``` +nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s remove +``` + +NOTE: This will remove all snapshots in Ceph associated to this PVC/PV! + +## Show Snapshot and Image details for a PVC/PV + +``` +nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s show +```