diff --git a/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl b/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl new file mode 100644 index 000000000..1cb27a52c --- /dev/null +++ b/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl @@ -0,0 +1,53 @@ +#!/bin/bash + +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +ARG=${1} + +if [ "x${ARG}" == "xcron" ]; then + PODS=$(kubectl get pods --namespace=${NAMESPACE} \ + --selector=application=ceph,component=osd --field-selector=status.phase=Running \ + '--output=jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}') + + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NAMESPACE} -- \ + sh -c -e "/tmp/utils-defragOSDs.sh defrag" + done +fi + +if [ "x${ARG}" == "xdefrag" ]; then + OSD_PATH=$(cat /proc/mounts | awk '/ceph-/{print $2}') + OSD_DEVICE=$(cat /proc/mounts | awk '/ceph-/{print $1}') + OSD_STORE=$(cat ${OSD_PATH}/type) + + ODEV=$(echo "${OSD_DEVICE}" | sed 's/\(.*[^0-9]\)[0-9]*$/\1/' | awk -F'/' '{print $3}') + ODEV_ROTATIONAL=$(cat /sys/block/${ODEV}/queue/rotational) + ODEV_SCHEDULER=$(cat /sys/block/${ODEV}/queue/scheduler) + + # NOTE(supamatt): TODO implement bluestore defrag options once it's available upstream + if [ "${ODEV_ROTATIONAL}" -eq "1" ] && [ "x${OSD_STORE}" == "xfilestore" ]; then + # NOTE(supamatt): Switch to CFQ in order to not block I/O + echo "cfq" | tee /sys/block/${ODEV}/queue/scheduler || true + ionice -c 3 xfs_fsr "${OSD_DEVICE}" 2>/dev/null + # NOTE(supamatt): Switch back to previous IO scheduler + echo ${ODEV_SCHEDULER} | tee /sys/block/${ODEV}/queue/scheduler || true + fi +fi + +exit 0 diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml index 7da5d63e1..6a3b93c7e 100644 --- a/ceph-osd/templates/configmap-bin.yaml +++ b/ceph-osd/templates/configmap-bin.yaml @@ -46,4 +46,6 @@ data: {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} utils-checkDNS.sh: | {{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-defragOSDs.sh: | +{{ tuple "bin/utils/_defragOSDs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ceph-osd/templates/cronjob-defragosds.yaml b/ceph-osd/templates/cronjob-defragosds.yaml new file mode 100644 index 000000000..d75e6c053 --- /dev/null +++ b/ceph-osd/templates/cronjob-defragosds.yaml @@ -0,0 +1,105 @@ +{{/* +Copyright 2018 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cronjob_defragosds }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-defragosds" }} +{{ tuple $envAll "ceph_defragosds" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - get + - list + - watch + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ $serviceAccountName }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.ceph_defragosds.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.successJob }} + failedJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.failJob }} + concurrencyPolicy: {{ .Values.jobs.ceph_defragosds.concurrency.execPolicy }} + startingDeadlineSeconds: {{ .Values.jobs.ceph_defragosds.startingDeadlineSecs }} + jobTemplate: + metadata: + labels: +{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.osd.node_selector_key }}: {{ .Values.labels.osd.node_selector_value }} + containers: + - name: {{ $serviceAccountName }} +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 12 }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBECTL_PARAM + value: {{ tuple $envAll "ceph" "ceph-defragosd" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }} + command: ["/tmp/utils-defragosds.sh"] + args: ["cron"] + volumeMounts: + - name: ceph-osd-bin + mountPath: /tmp/utils-defragosds.sh + subPath: utils-defragosds.sh + readOnly: true + restartPolicy: Never + hostNetwork: true + volumes: + - name: ceph-osd-bin + configMap: + name: ceph-osd-bin + defaultMode: 0555 + - name: ceph-osd-etc + configMap: + name: ceph-osd-etc + defaultMode: 0444 +{{- end }} diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml index 9bc179376..89afc2de8 100644 --- a/ceph-osd/templates/daemonset-osd.yaml +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -272,6 +272,10 @@ spec: mountPath: /tmp/osd-common.sh subPath: osd-common.sh readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-defragOSDs.sh + subPath: utils-defragOSDs.sh + readOnly: true - name: ceph-osd-etc mountPath: /etc/ceph/storage.json subPath: storage.json diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml index 6ae9ebc64..f031eee2d 100644 --- a/ceph-osd/values.yaml +++ b/ceph-osd/values.yaml @@ -76,10 +76,25 @@ secrets: keyrings: osd: ceph-bootstrap-osd-keyring admin: ceph-client-admin-keyring + network: public: 192.168.0.0/16 cluster: 192.168.0.0/16 +jobs: + ceph_defragosds: + # Execute the 1st of each month + cron: "0 0 1 * *" + history: + # Number of successful job to keep + successJob: 1 + # Number of failed job to keep + failJob: 1 + concurrency: + # Skip new job if previous job still active + execPolicy: Forbid + startingDeadlineSecs: 60 + conf: ceph: global: @@ -250,3 +265,4 @@ manifests: daemonset_osd: true job_image_repo_sync: true helm_tests: true + cronjob_defragosds: true