openstack/openstack-helm-infra
Code Issues Proposed changes
openstack-helm-infra/ceph-osd/templates/daemonset-osd.yaml
Matthew Heler 6e8c289c13 Add failure domains, and device classes for custom CRUSH rules 
Largely inspired and taken from Kranthi's PS.

 - Add support for creating custom CRUSH rules based off of failure
domains and device classes (ssd & hdd)
- Basic logic around the PG calculator to autodetect the number of
OSDs globally and per device class (required when using custom crush
rules that specify device classes).

Change-Id: I13a6f5eb21494746c2b77e340e8d0dcb0d81a591
2018-11-27 09:37:30 -06:00

306 lines
11 KiB
YAML
Raw Blame History

{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "ceph.osd.daemonset" }}
{{- $daemonset := index . 0 }}
{{- $configMapName := index . 1 }}
{{- $serviceAccountName := index . 2 }}
{{- $envAll := index . 3 }}
{{- with $envAll }}
---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: ceph-osd
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
labels:
{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
spec:
selector:
matchLabels:
{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }}
template:
metadata:
labels:
{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
nodeSelector:
{{ .Values.labels.osd.node_selector_key }}: {{ .Values.labels.osd.node_selector_value }}
hostNetwork: true
hostPID: true
dnsPolicy: {{ .Values.pod.dns_policy }}
initContainers:
{{ tuple $envAll "osd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
- name: ceph-init-dirs
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
command:
- /tmp/init-dirs.sh
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render.
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
volumeMounts:
- name: ceph-osd-bin
mountPath: /tmp/init-dirs.sh
subPath: init-dirs.sh
readOnly: true
- name: pod-var-lib-ceph
mountPath: /var/lib/ceph
readOnly: false
- name: pod-run
mountPath: /run
readOnly: false
- name: ceph-log-ownership
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
securityContext:
runAsUser: 0
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render and added to all containers in the
# pod
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
- name: CEPH_GET_ADMIN_KEY
value: "1"
command:
- chown
- -R
- ceph:root
- /var/log/ceph
volumeMounts:
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
- name: osd-init
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
privileged: true
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render and added to all containers in the
# pod
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
- name: CEPH_GET_ADMIN_KEY
value: "1"
command:
- /tmp/osd-init.sh
volumeMounts:
- name: ceph-osd-bin
mountPath: /tmp/osd-init.sh
subPath: osd-init.sh
readOnly: true
- name: ceph-osd-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: devices
mountPath: /dev
readOnly: false
- name: pod-var-lib-ceph
mountPath: /var/lib/ceph
readOnly: false
- name: pod-run
mountPath: /run
readOnly: false
- name: run-lvm
mountPath: /run/lvm
readOnly: false
- name: data
mountPath: /var/lib/ceph/osd
readOnly: false
- name: journal
mountPath: /var/lib/ceph/journal
readOnly: false
containers:
- name: osd-pod
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
privileged: true
env:
# NOTE(portdirect): These environment variables will be populated
# dynamicly at the point of render.
# - name: JOURNAL_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/journal-one
# - name: STORAGE_LOCATION
# value: /var/lib/openstack-helm/ceph/osd/data-one
# - name: JOURNAL_TYPE
# value: directory
# - name: STORAGE_TYPE
# value: directory
- name: CLUSTER
value: "ceph"
- name: CEPH_GET_ADMIN_KEY
value: "1"
- name: CRUSH_FAILURE_DOMAIN_TYPE
value: {{ .Values.conf.storage.failure_domain | default "host" | quote }}
- name: CRUSH_FAILURE_DOMAIN_NAME
value: {{ .Values.conf.storage.failure_domain_name | default "false" | quote }}
- name: CRUSH_FAILURE_DOMAIN_BY_HOSTNAME
value: {{ .Values.conf.storage.failure_domain_by_hostname | default "false" | quote }}
command:
- /tmp/osd-start.sh
lifecycle:
preStop:
exec:
command:
- /tmp/osd-stop.sh
livenessProbe:
exec:
command:
- /tmp/osd-check.sh
- liveness
initialDelaySeconds: 120
periodSeconds: 60
readinessProbe:
exec:
command:
- /tmp/osd-check.sh
- readiness
initialDelaySeconds: 60
periodSeconds: 60
volumeMounts:
- name: ceph-osd-bin
mountPath: /tmp/osd-start.sh
subPath: osd-start.sh
readOnly: true
- name: ceph-osd-bin
mountPath: /tmp/osd-directory.sh
subPath: osd-directory.sh
readOnly: true
- name: ceph-osd-bin
mountPath: /tmp/osd-block.sh
subPath: osd-block.sh
readOnly: true
- name: ceph-osd-bin
mountPath: /tmp/osd-check.sh
subPath: osd-check.sh
readOnly: true
- name: ceph-osd-bin
mountPath: /tmp/osd-stop.sh
subPath: osd-stop.sh
readOnly: true
- name: ceph-osd-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: devices
mountPath: /dev
readOnly: false
- name: pod-var-lib-ceph
mountPath: /var/lib/ceph
readOnly: false
- name: pod-run
mountPath: /run
readOnly: false
- name: run-lvm
mountPath: /run/lvm
readOnly: false
- name: data
mountPath: /var/lib/ceph/osd
readOnly: false
- name: journal
mountPath: /var/lib/ceph/journal
readOnly: false
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
volumes:
- name: devices
hostPath:
path: /dev
- name: run-lvm
hostPath:
path: /run/lvm
- name: pod-var-lib-ceph
emptyDir: {}
- name: pod-var-log
hostPath:
path: {{ print "/var/log/ceph/" $envAll.Release.Name }}
- name: pod-run
emptyDir:
medium: "Memory"
- name: ceph-osd-bin
configMap:
name: ceph-osd-bin
defaultMode: 0555
- name: ceph-osd-etc
configMap:
name: {{ $configMapName }}
defaultMode: 0444
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
# NOTE(portdirect): If directory mounts are to be used for OSD's
# they will automaticly be inserted here, with the format:
# - name: data
# hostPath:
# path: /var/lib/foo
# - name: journal
# hostPath:
# path: /var/lib/bar
{{- end }}
{{- end }}
{{- if .Values.manifests.daemonset_osd }}
{{- $daemonset := "osd" }}
{{- $configMapName := "ceph-osd-etc" }}
{{- $serviceAccountName := "ceph-osd"}}
{{ tuple . "osd" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName . | include "ceph.osd.daemonset" | toString | fromYaml }}
{{- $configmap_yaml := "ceph.osd.configmap.etc" }}
{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.osd_daemonset_overrides" }}
{{- end }}
View Git Blame Copy Permalink