[CEPH] Add a support script for manipulating a PV backed by RBD

Features of the script include:
- creating and removing a snapshot within Ceph for a PV
- backing up and restoring a PV within Ceph
- listing the RBD image details (watchers, thin/thick provisioned
usage, image properties) and snapshot information

Take note that all backups are by default stored locally in
"/var/lib/openstack-helm/ceph/backup" on the POD hosting this
utility container

Change-Id: I5de30d5337754a411b5e1162a215596afb469bac

Dockerfiles/ceph-utility/Dockerfile.ubuntu

@@ -1,26 +1,24 @@
-ARG UBUNTU_RELEASE=xenial
+ARG FROM=docker.io/ubuntu:xenial
-
+FROM ${FROM}
-FROM docker.io/ubuntu:${UBUNTU_RELEASE}
-LABEL maintainer="sreejith.punnapuzha@outlook.com"
 
 ARG CEPH_RELEASE=mimic
-ARG UBUNTU_RELEASE=xenial
+ARG KUBE_VERSION=1.12.2
 
 RUN set -xe \
-    && echo '#!/bin/sh' > /usr/sbin/policy-rc.d \
+    && export DEBIAN_FRONTEND=noninteractive \
-    && echo 'exit 101' >> /usr/sbin/policy-rc.d \
-    && chmod +x /usr/sbin/policy-rc.d \
     && sed -i '/nobody/d' /etc/passwd \
     && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \
-    && dpkg-divert --local --rename --add /sbin/initctl \
+    && apt-get update && apt-get dist-upgrade -y \
-    && cp -a /usr/sbin/policy-rc.d /sbin/initctl \
+    && apt-get install -y wget curl apt-transport-https ca-certificates gnupg\
-    && sed -i 's/^exit.*/exit 0/' /sbin/initctl \
-    && apt-get update && apt-get install -y wget curl apt-transport-https gnupg\
     && wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - \
-    && echo deb https://download.ceph.com/debian-${CEPH_RELEASE}/ ${UBUNTU_RELEASE} main | tee /etc/apt/sources.list.d/ceph.list \
+    && echo deb https://download.ceph.com/debian-${CEPH_RELEASE}/ xenial main | tee /etc/apt/sources.list.d/ceph.list \
-    && apt-get update && apt-get install -y bash python-oslo.rootwrap moreutils vim sudo screen ceph-common python-rbd radosgw rsyslog x11-apps jq \
+    && apt-get update && apt-get install -y bash python-oslo.rootwrap moreutils vim sudo screen ceph ceph-common python-rbd radosgw rsyslog hexedit jq s3cmd rsync xz-utils iperf \
     && apt-get remove --purge -y wget apt-transport-https && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* \
+    && TMP_DIR=$(mktemp --directory) \
+    && cd ${TMP_DIR} \
+    && curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
     && curl -sSL https://bootstrap.pypa.io/get-pip.py | python \
-    && pip --no-cache-dir install --upgrade crush
+    && rm -rf ${TMP_DIR}
-
 CMD ["/bin/bash"]

ceph-utility/templates/bin/utility/_nccli.tpl

@@ -14,4 +14,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */}}
-script -f -a -q /var/log/syslog -c "sudo /usr/local/bin/ceph-utility-rootwrap /etc/ceph/rootwrap.conf $*"
+script -f -a -q /var/log/syslog -c "sudo -E /usr/local/bin/ceph-utility-rootwrap /etc/ceph/rootwrap.conf $*"

ceph-utility/templates/bin/utility/_rbd_pv.tpl

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+{{/*
+Copyright 2019 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+usage() {
+  echo "Backup Usage: nccli rbd_pv [-b <pvc name>] [-n <namespace>] [-d <backup dest> (optional, default: /backup)] [-p <ceph rbd pool> (optional, default: rbd)]"
+  echo "Restore Usage: nccli rbd_pv [-r <restore_file>] [-p <ceph rbd pool> (optional, default: rbd)]"
+  echo "Snapshot Usage: nccli rbd_pv [-b <pvc name>] [-n <namespace>] [-p <ceph rbd pool> (optional, default: rbd] [-s <create|rollback|remove|show> (required) ]"
+  exit 1
+}
+
+while getopts ":b:p:n:d:r:h:s:" opt; do
+  case $opt
+  in
+    b) pvc_name=${OPTARG};;
+    n) nspace=${OPTARG};;
+    d) backup_dest=${OPTARG};;
+    r) restore_file=${OPTARG};;
+    p) rbd_pool=${OPTARG};;
+    s) snapshot=${OPTARG};;
+    h) usage ;;
+  esac
+done
+
+if [[ -z "${pvc_name}" || -z "${nspace}" ]]; then
+  if [[ -z "${restore_file}" ]]; then
+    usage
+    echo "ERROR: Missing command line arguement(s)!"
+    exit 1
+  fi
+fi
+
+if [[ -z "${rbd_pool}" ]]; then
+  rbd_pool="rbd"
+fi
+
+timestamp="$(date +%F_%T)"
+
+if [[ ! -z "${restore_file}" ]]; then
+  if [[ -e "${restore_file}" ]]; then
+    rbd_image="$(echo "${restore_file}" | rev | awk -v FS='/' '{print $1}' | rev | cut -f 1 -d '.')"
+    if (nccli rbd info "${rbd_pool}"/"${rbd_image}" | grep -q id); then
+      nccli rbd mv ${rbd_pool}/${rbd_image} ${rbd_pool}/${rbd_image}.orig-${timestamp}
+      echo "WARNING: Existing PVC/RBD image has been moved to ${rbd_pool}/${rbd_image}.orig-${timestamp}"
+    fi
+    nccli rbd import ${restore_file} ${rbd_pool}/${rbd_image}
+    echo "INFO: Backup has been restored into ${rbd_pool}/${rbd_image}"
+  else
+    echo "ERROR: Missing restore file!"
+    exit 1
+  fi
+elif [[ ! -z "${snapshot}" ]]; then
+  volume="$(kubectl -n ${nspace} get pvc ${pvc_name} --no-headers | awk '{ print $3 }')"
+  rbd_image="$(kubectl get pv "${volume}" -o json | jq -r '.spec.rbd.image')"
+
+  if [[ "x${snapshot}x" == "xcreatex" ]]; then
+    snap_name="${pvc_name}-${timestamp}"
+    nccli rbd snap create ${rbd_pool}/${rbd_image}@${snap_name}
+    echo "INFO: Snapshot ${rbd_pool}/${rbd_image}@${snap_name} has been created for PVC ${pvc_name}"
+  elif [[ "x${snapshot}x" == "xrollback" ]]; then
+    snap_name=$(nccli rbd snap ls ${rbd_pool}/${rbd_image})
+    nccli rbd snap rollback ${rbd_pool}/${rbd_image}@${snap_name}
+    echo "WARNING: Rolled back snapshot ${rbd_pool}/${rbd_image}@${snap_name} for ${pvc_name}"
+  elif [[ "x${snapshot}x" == "xremovex" ]]; then
+    nccli rbd snap purge ${rbd_pool}/${rbd_image}
+    echo "Removed snapshot(s) for ${pvc_name}"
+  elif [[ "x${snapshot}x" == "xshowx" ]]; then
+    echo "INFO: This PV is mapped to the following RBD Image:"
+    echo "${rbd_pool}/${rbd_image}"
+    echo -e "\nINFO: Current open sessions to RBD Image:"
+    nccli rbd status ${rbd_pool}/${rbd_image}
+    echo -e "\nINFO: RBD Image information:"
+    nccli rbd info ${rbd_pool}/${rbd_image}
+    echo -e "\nINFO: RBD Image snapshot details:"
+    rbd snap ls ${rbd_pool}/${rbd_image}
+    echo -e "\nINFO: RBD Image size details:"
+    nccli rbd du ${rbd_pool}/${rbd_image}
+  else
+    echo "ERROR: Missing arguement for snapshot option!"
+  fi
+else
+  if [[ -z "${backup_dest}" ]]; then
+    backup_dest="/backup"
+  fi
+  if [[ ! -d "${backup_dest}" ]]; then
+    echo "ERROR: Backup destination does not exist, cannot continue with the backup!"
+    exit 1
+  fi
+
+  echo "INFO: Backing up ${pvc_name} within namespace ${nspace}"
+  volume="$(kubectl -n ${nspace} get pvc ${pvc_name} --no-headers | awk '{ print $3 }')"
+  rbd_image="$(kubectl get pv "${volume}" -o json | jq -r '.spec.rbd.image')"
+
+  if [[ -z "${volume}" ]] || (! nccli rbd info "${rbd_pool}"/"${rbd_image}" | grep -q id); then
+    echo "ERROR: PVC does not exist or is missing! Cannot continue with backup for ${pvc_name}"
+    exit 1
+  else
+    # Create current snapshot and export to a file
+    snap_name="${pvc_name}-${timestamp}"
+    backup_name="${rbd_image}.${pvc_name}-${timestamp}"
+    nccli rbd snap create ${rbd_pool}/${rbd_image}@${snap_name}
+    nccli rbd export ${rbd_pool}/${rbd_image}@${snap_name} ${backup_dest}/${backup_name}
+    # Remove snapshot otherwise we may see an issue cleaning up the PVC from K8s, and from Ceph.
+    nccli rbd snap rm ${rbd_pool}/${rbd_image}@${snap_name}
+    echo "INFO: PV ${pvc_name} saved to:"
+    echo "${backup_dest}/${backup_name}"
+  fi
+fi
+
+exit 0

ceph-utility/templates/configmap-bin.yaml

@@ -44,4 +44,7 @@ data:
   osd-maintenance: |
 {{ tuple "bin/utility/_osd-maintenance.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
 
+  rbd_pv: |
+{{ tuple "bin/utility/_rbd_pv.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
 {{- end }}

ceph-utility/templates/deployment-utility.yaml

@@ -20,6 +20,35 @@ limitations under the License.
 {{- $serviceAccountName := printf "%s" $envAll.Release.Name }}
 {{ tuple $envAll "utility" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: {{ $serviceAccountName }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - persistentvolumeclaims
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $serviceAccountName }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ $serviceAccountName }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $serviceAccountName }}
+    namespace: {{ $envAll.Release.Namespace }}
+---
 kind: Deployment
 apiVersion: apps/v1
 metadata:
@@ -71,6 +100,10 @@ spec:
               mountPath: /tmp/osd-maintenance
               subPath: osd-maintenance
               readOnly: true
+            - name: ceph-utility-bin
+              mountPath: /tmp/rbd_pv
+              subPath: rbd_pv
+              readOnly: true
             - name: ceph-utility-sudoers
               mountPath: /etc/sudoers.d/nccli-sudo
               subPath: nccli-sudo
@@ -90,6 +123,9 @@ spec:
               mountPath: /etc/ceph/rootwrap.conf
               subPath: rootwrap.conf
               readOnly: true
+            - name: pod-ceph-backup
+              mountPath: /backup
+              readOnly: false
 
       volumes:
         - name: ceph-utility-sudoers
@@ -112,4 +148,8 @@ spec:
           secret:
             secretName: {{ .Values.secrets.keyrings.admin | quote }}
             defaultMode: 0600
+        - name: pod-ceph-backup
+          hostPath:
+            path: {{ .Values.conf.storage.utility.backup_target }}
+            type: DirectoryOrCreate
 {{- end }}

ceph-utility/values.yaml

@@ -25,7 +25,7 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_utility: 'docker.io/sreejithpunnapuzha/ceph-utility:v0.0.2'
+    ceph_utility: 'docker.io/sreejithpunnapuzha/ceph-utility:v0.0.3'
     image_repo_sync: docker.io/docker:17.07.0
   local_registry:
     active: false
@@ -92,6 +92,8 @@ conf:
       radosgw-admin: CommandFilter, radosgw-admin, root
       rbd: CommandFilter, rbd, root
       osd-maintenance: CommandFilter, osd-maintenance, root
+      rbd_pv: CommandFilter, rbd_pv, root
+      kubectl: CommandFilter, kubectl, root
       # Below are examples of RegExpFilter. This will restict access to ceph cluster even with admin user
       #rbd00: RegExpFilter, rbd, root, rbd, (^((?!clone|copy|cp|create|export|export-diff|flatten|import|import-diff|map|merge-diff|pool|remove|rm|rename|mv|resize|unmap).)*$)
       #rbd01: RegExpFilter, rbd, root, rbd, image-meta, (^((?!get|remove|set).)*$)
@@ -126,6 +128,9 @@ conf:
       # INFO means log all usage
       # ERROR means only log unsuccessful attempts
       syslog_log_level: INFO
+  storage:
+    utility:
+      backup_target: /var/lib/openstack-helm/ceph/backup
 
 dependencies:
   dynamic:

docs/rbd_pv.md

@@ -0,0 +1,58 @@
+# RBD PVC/PV script
+
+This MOP covers Maintenance Activities related to using the rbd_pv script
+to backup and recover PVCs within your kubernetes environment using Ceph.
+
+## Usage
+Execute nccli rbd_pv without arguements to list usage options.
+
+```
+nccli rbd_pv
+Backup Usage: nccli rbd_pv [-b <pvc name>] [-n <namespace>] [-d <backup dest> (optional, default: /tmp/backup)] [-p <ceph rbd pool> (optional, default: rbd)]
+Restore Usage: nccli rbd_pv [-r <restore_file>] [-p <ceph rbd pool> (optional, default: rbd)]
+Snapshot Usage: nccli rbd_pv [-b <pvc name>] [-n <namespace>] [-p <ceph rbd pool> (optional, default: rbd] [-s <create|rollback|remove> (required)]
+```
+
+## Backing up a PVC/PV from RBD
+To backup a PV, execute the following:
+
+```
+nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack
+```
+
+## Restoring a PVC/PV backup
+To restore a PV RBD backup image, execute the following:
+
+```
+nccli rbd_pv -r /backup/kubernetes-dynamic-pvc-ab1f2e8f-21a4-11e9-ab61-ca77944df03c.img
+```
+NOTE: The original PVC/PV will be renamed and not overwritten.
+NOTE: Before restoring, you _must_ ensure it is not mounted!
+
+## Creating a Snapshot for a PVC/PV
+
+```
+nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s create
+```
+
+## Rolling back to a Snapshot for a PVC/PV
+
+```
+nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s rollback
+```
+
+NOTE: Before rolling back a snapshot, you _must_ ensure the PVC/PV volume is not mounted!!
+
+## Removing a Snapshot for a PVC/PV
+
+```
+nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s remove
+```
+
+NOTE: This will remove all snapshots in Ceph associated to this PVC/PV!
+
+## Show Snapshot and Image details for a PVC/PV
+
+```
+nccli rbd_pv -b mysql-data-mariadb-server-0 -n openstack -s show
+```