From ccea6b4df564235d63b1fe89b3ac30548b096798 Mon Sep 17 00:00:00 2001 From: Alan Meadows Date: Tue, 13 Jun 2017 17:50:54 -0700 Subject: [PATCH] Convert ceph to utilize hostNetworking This enhances the stability and recovery of ceph by leveraging hostNetworking for monitors as well as OSDs, and enables the K8S_HOST_NETWORK variable within ceph-docker. This enhancement should allow recovery of monitors from a complete downed cluster. Additionally, ceph's generic "ceph-storage" node label has been split out for mon, mds, and osd. Co-Authored-By: Larry Rensing Change-Id: I27efe5c41d04ab044dccb5f38db897cb041d4723 --- ceph/templates/bin/_config.k8s.sh.tpl | 39 +++++++++ .../bin/_variables_entrypoint.sh.tpl | 85 +++++++++++++++++++ ceph/templates/configmap-bin.yaml | 5 ++ ...tatefulset-mon.yaml => daemonset-mon.yaml} | 57 ++++--------- ceph/templates/daemonset-osd.yaml | 21 ++--- ceph/templates/deployment-mds.yaml | 20 ++--- ceph/templates/deployment-moncheck.yaml | 21 ++--- ceph/templates/deployment-rgw.yaml | 21 ++--- ceph/templates/job-bootstrap.yaml | 2 +- .../{service.yaml => service-mon.yaml} | 7 +- ceph/templates/service-rgw.yaml | 7 +- .../{storage.yaml => storageclass.yaml} | 0 ceph/values.yaml | 24 ++++-- doc/source/install/multinode.rst | 27 ++++-- mariadb/values.yaml | 2 +- tools/gate/basic_launch.sh | 16 ++-- tools/gate/funcs/network.sh | 18 +++- 17 files changed, 239 insertions(+), 133 deletions(-) create mode 100644 ceph/templates/bin/_config.k8s.sh.tpl create mode 100644 ceph/templates/bin/_variables_entrypoint.sh.tpl rename ceph/templates/{statefulset-mon.yaml => daemonset-mon.yaml} (76%) rename ceph/templates/{service.yaml => service-mon.yaml} (92%) rename ceph/templates/{storage.yaml => storageclass.yaml} (100%) diff --git a/ceph/templates/bin/_config.k8s.sh.tpl b/ceph/templates/bin/_config.k8s.sh.tpl new file mode 100644 index 0000000000..a830c4e178 --- /dev/null +++ b/ceph/templates/bin/_config.k8s.sh.tpl @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +function get_admin_key { + # No-op for static + log "k8s: does not generate the admin key. Use Kubernetes secrets instead." +} + +function get_mon_config { + # Get fsid from ceph.conf + local fsid=$(ceph-conf --lookup fsid -c /etc/ceph/${CLUSTER}.conf) + + timeout=10 + MONMAP_ADD="" + + while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do + # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params + if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then + MONMAP_ADD=$(kubectl get pods --namespace=${CLUSTER} -l application=ceph -l component=mon -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") + else + MONMAP_ADD=$(kubectl get pods --namespace=${CLUSTER} -l application=ceph -l component=mon -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}") + fi + (( timeout-- )) + sleep 1 + done + + if [[ -z "${MONMAP_ADD// }" ]]; then + exit 1 + fi + + # Create a monmap with the Pod Names and IP + monmaptool --create ${MONMAP_ADD} --fsid ${fsid} $MONMAP --clobber + +} + +function get_config { + # No-op for static + log "k8s: config is stored as k8s secrets." +} diff --git a/ceph/templates/bin/_variables_entrypoint.sh.tpl b/ceph/templates/bin/_variables_entrypoint.sh.tpl new file mode 100644 index 0000000000..f8a75deb58 --- /dev/null +++ b/ceph/templates/bin/_variables_entrypoint.sh.tpl @@ -0,0 +1,85 @@ +########################################## +# LIST OF ALL DAEMON SCENARIOS AVAILABLE # +########################################## + +ALL_SCENARIOS="populate_kvstore mon osd osd_directory osd_directory_single osd_ceph_disk osd_ceph_disk_prepare osd_ceph_disk_activate osd_ceph_activate_journal mds rgw rgw_user restapi nfs zap_device mon_health" + + +######################### +# LIST OF ALL VARIABLES # +######################### + +: ${CLUSTER:=ceph} +: ${CLUSTER_PATH:=ceph-config/${CLUSTER}} # For KV config +: ${CEPH_CLUSTER_NETWORK:=${CEPH_PUBLIC_NETWORK}} +: ${CEPH_DAEMON:=${1}} # default daemon to first argument +: ${CEPH_GET_ADMIN_KEY:=0} +: ${HOSTNAME:=$(uname -n)} +: ${MON_NAME:=${HOSTNAME}} +# (openstack-helm): we need the MONMAP to be stateful, so we retain it +: ${MONMAP=/etc/ceph/monmap-${CLUSTER}} +: ${MON_DATA_DIR:=/var/lib/ceph/mon/${CLUSTER}-${MON_NAME}} +: ${K8S_HOST_NETWORK:=0} +: ${NETWORK_AUTO_DETECT:=0} +: ${MDS_NAME:=mds-${HOSTNAME}} +: ${OSD_FORCE_ZAP:=0} +: ${OSD_JOURNAL_SIZE:=100} +: ${OSD_BLUESTORE:=0} +: ${OSD_DMCRYPT:=0} +: ${OSD_JOURNAL_UUID:=$(uuidgen)} +: ${OSD_LOCKBOX_UUID:=$(uuidgen)} +: ${CRUSH_LOCATION:=root=default host=${HOSTNAME}} +: ${CEPHFS_CREATE:=0} +: ${CEPHFS_NAME:=cephfs} +: ${CEPHFS_DATA_POOL:=${CEPHFS_NAME}_data} +: ${CEPHFS_DATA_POOL_PG:=8} +: ${CEPHFS_METADATA_POOL:=${CEPHFS_NAME}_metadata} +: ${CEPHFS_METADATA_POOL_PG:=8} +: ${RGW_NAME:=${HOSTNAME}} +: ${RGW_ZONEGROUP:=} +: ${RGW_ZONE:=} +: ${RGW_CIVETWEB_PORT:=8080} +: ${RGW_REMOTE_CGI:=0} +: ${RGW_REMOTE_CGI_PORT:=9000} +: ${RGW_REMOTE_CGI_HOST:=0.0.0.0} +: ${RGW_USER:="cephnfs"} +: ${RESTAPI_IP:=0.0.0.0} +: ${RESTAPI_PORT:=5000} +: ${RESTAPI_BASE_URL:=/api/v0.1} +: ${RESTAPI_LOG_LEVEL:=warning} +: ${RESTAPI_LOG_FILE:=/var/log/ceph/ceph-restapi.log} +: ${KV_TYPE:=none} # valid options: etcd, k8s|kubernetes or none +: ${KV_IP:=127.0.0.1} +: ${KV_PORT:=4001} +: ${GANESHA_OPTIONS:=""} +: ${GANESHA_EPOCH:=""} # For restarting + +# This is ONLY used for the CLI calls, e.g: ceph $CLI_OPTS health +CLI_OPTS="--cluster ${CLUSTER}" + +# This is ONLY used for the daemon's startup, e.g: ceph-osd $DAEMON_OPTS +DAEMON_OPTS="--cluster ${CLUSTER} --setuser ceph --setgroup ceph -d" + +MOUNT_OPTS="-t xfs -o noatime,inode64" +ETCDCTL_OPTS="--peers ${KV_IP}:${KV_PORT}" + +# make sure etcd uses http or https as a prefix +if [[ "$KV_TYPE" == "etcd" ]]; then + if [ -n "${KV_CA_CERT}" ]; then + CONFD_NODE_SCHEMA="https://" + KV_TLS="--ca-file=${KV_CA_CERT} --cert-file=${KV_CLIENT_CERT} --key-file=${KV_CLIENT_KEY}" + CONFD_KV_TLS="-scheme=https -client-ca-keys=${KV_CA_CERT} -client-cert=${KV_CLIENT_CERT} -client-key=${KV_CLIENT_KEY}" + else + CONFD_NODE_SCHEMA="http://" + fi +fi + +# Internal variables +MDS_KEYRING=/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}/keyring +ADMIN_KEYRING=/etc/ceph/${CLUSTER}.client.admin.keyring +MON_KEYRING=/etc/ceph/${CLUSTER}.mon.keyring +RGW_KEYRING=/var/lib/ceph/radosgw/${RGW_NAME}/keyring +MDS_BOOTSTRAP_KEYRING=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring +RGW_BOOTSTRAP_KEYRING=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring +OSD_BOOTSTRAP_KEYRING=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring +OSD_PATH_BASE=/var/lib/ceph/osd/${CLUSTER} diff --git a/ceph/templates/configmap-bin.yaml b/ceph/templates/configmap-bin.yaml index 83b9f2e77b..7ae4de1b4a 100644 --- a/ceph/templates/configmap-bin.yaml +++ b/ceph/templates/configmap-bin.yaml @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. --- + apiVersion: v1 kind: ConfigMap metadata: @@ -35,6 +36,8 @@ data: {{- end }} common_functions.sh: |+ {{ tuple "bin/_common_functions.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + config.k8s.sh: | +{{ tuple "bin/_config.k8s.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} entrypoint.sh: |+ {{ tuple "bin/_entrypoint.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} osd_activate_journal.sh: |+ @@ -63,3 +66,5 @@ data: {{ tuple "bin/_start_rgw.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} watch_mon_health.sh: |+ {{ tuple "bin/_watch_mon_health.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + variables_entrypoint.sh: | +{{ tuple "bin/_variables_entrypoint.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/daemonset-mon.yaml similarity index 76% rename from ceph/templates/statefulset-mon.yaml rename to ceph/templates/daemonset-mon.yaml index 8111f9fa1b..0eddfd5e28 100644 --- a/ceph/templates/statefulset-mon.yaml +++ b/ceph/templates/daemonset-mon.yaml @@ -16,63 +16,34 @@ {{- $envAll := . }} {{- $dependencies := .Values.dependencies.mon }} --- -apiVersion: apps/v1beta1 -kind: StatefulSet +kind: DaemonSet +apiVersion: extensions/v1beta1 metadata: - labels: - app: ceph - daemon: mon name: ceph-mon spec: - serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - replicas: {{ .Values.replicas.mon }} template: metadata: - name: ceph-mon labels: - app: ceph - daemon: mon +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: - # alanmeadows: this soft requirement allows single - # host deployments to spawn several ceph-mon - # containers - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - ceph - - key: daemon - operator: In - values: - - mon - topologyKey: kubernetes.io/hostname - weight: 10 nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.mon.node_selector_key }}: {{ .Values.labels.mon.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + hostNetwork: true serviceAccount: default containers: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - {{- if .Values.pod.resources.enabled }} - resources: - requests: - memory: {{ .Values.pod.resources.mon.requests.memory | quote }} - cpu: {{ .Values.pod.resources.mon.requests.cpu | quote }} - limits: - memory: {{ .Values.pod.resources.mon.limits.memory | quote }} - cpu: {{ .Values.pod.resources.mon.limits.cpu | quote }} - {{- end }} +{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - containerPort: 6789 env: + - name: K8S_HOST_NETWORK + value: "1" + - name: MONMAP + value: /var/lib/ceph/mon/monmap - name: CEPH_DAEMON value: MON - name: KV_TYPE @@ -124,6 +95,14 @@ spec: mountPath: /etc/ceph/ceph.mon.keyring subPath: ceph.mon.keyring readOnly: false + - name: ceph-bin + mountPath: /variables_entrypoint.sh + subPath: variables_entrypoint.sh + readOnly: true + - name: ceph-bin + mountPath: /config.k8s.sh + subPath: config.k8s.sh + readOnly: true - name: ceph-bootstrap-osd-keyring mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring subPath: ceph.keyring diff --git a/ceph/templates/daemonset-osd.yaml b/ceph/templates/daemonset-osd.yaml index 5d611b7676..d832486d58 100644 --- a/ceph/templates/daemonset-osd.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -20,36 +20,27 @@ kind: DaemonSet apiVersion: extensions/v1beta1 metadata: name: ceph-osd - labels: - app: ceph - daemon: osd spec: template: metadata: labels: - app: ceph - daemon: osd +{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.osd.node_selector_key }}: {{ .Values.labels.osd.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + hostNetwork: true containers: - name: osd-pod image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - {{- if .Values.pod.resources.enabled }} - resources: - requests: - memory: {{ .Values.pod.resources.osd.requests.memory | quote }} - cpu: {{ .Values.pod.resources.osd.requests.cpu | quote }} - limits: - memory: {{ .Values.pod.resources.osd.limits.memory | quote }} - cpu: {{ .Values.pod.resources.osd.limits.cpu | quote }} - {{- end }} +{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} securityContext: privileged: true env: + - name: K8S_HOST_NETWORK + value: "1" - name: CEPH_DAEMON value: osd_directory - name: KV_TYPE diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml index 201dbdb590..c2dfd4071d 100644 --- a/ceph/templates/deployment-mds.yaml +++ b/ceph/templates/deployment-mds.yaml @@ -19,9 +19,6 @@ kind: Deployment apiVersion: apps/v1beta1 metadata: - labels: - app: ceph - daemon: mds name: ceph-mds spec: replicas: 1 @@ -29,11 +26,10 @@ spec: metadata: name: ceph-mds labels: - app: ceph - daemon: mds +{{ tuple $envAll "ceph" "mds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.mds.node_selector_key }}: {{ .Values.labels.mds.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default @@ -41,18 +37,12 @@ spec: - name: ceph-mds image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - {{- if .Values.pod.resources.enabled }} - resources: - requests: - memory: {{ .Values.pod.resources.mds.requests.memory | quote }} - cpu: {{ .Values.pod.resources.mds.requests.cpu | quote }} - limits: - memory: {{ .Values.pod.resources.mds.limits.memory | quote }} - cpu: {{ .Values.pod.resources.mds.limits.cpu | quote }} - {{- end }} +{{ tuple $envAll $envAll.Values.pod.resources.mds | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - containerPort: 6800 env: + - name: K8S_HOST_NETWORK + value: "1" - name: CEPH_DAEMON value: MDS - name: CEPHFS_CREATE diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml index 371abe7016..df3d17efdc 100644 --- a/ceph/templates/deployment-moncheck.yaml +++ b/ceph/templates/deployment-moncheck.yaml @@ -19,21 +19,16 @@ kind: Deployment apiVersion: apps/v1beta1 metadata: - labels: - app: ceph - daemon: moncheck name: ceph-mon-check spec: replicas: {{ .Values.replicas.mon_check }} template: metadata: - name: ceph-mon labels: - app: ceph - daemon: moncheck +{{ tuple $envAll "ceph" "moncheck" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.mon.node_selector_key }}: {{ .Values.labels.mon.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default @@ -41,18 +36,12 @@ spec: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - {{- if .Values.pod.resources.enabled }} - resources: - requests: - memory: {{ .Values.pod.resources.mon_check.requests.memory | quote }} - cpu: {{ .Values.pod.resources.mon_check.requests.cpu | quote }} - limits: - memory: {{ .Values.pod.resources.mon_check.limits.memory | quote }} - cpu: {{ .Values.pod.resources.mon_check.limits.cpu | quote }} - {{- end }} +{{ tuple $envAll $envAll.Values.pod.resources.moncheck | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - containerPort: 6789 env: + - name: K8S_HOST_NETWORK + value: "1" - name: CEPH_DAEMON value: MON_HEALTH - name: KV_TYPE diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml index 774fbe507a..5ab2648c90 100644 --- a/ceph/templates/deployment-rgw.yaml +++ b/ceph/templates/deployment-rgw.yaml @@ -20,21 +20,16 @@ kind: Deployment apiVersion: apps/v1beta1 metadata: - labels: - app: ceph - daemon: rgw name: ceph-rgw spec: replicas: {{ .Values.replicas.rgw }} template: metadata: - name: ceph-rgw labels: - app: ceph - daemon: rgw +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} spec: nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.rgw.node_selector_key }}: {{ .Values.labels.rgw.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default @@ -42,18 +37,12 @@ spec: - name: ceph-rgw image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - {{- if .Values.pod.resources.enabled }} - resources: - requests: - memory: {{ .Values.pod.resources.rgw.requests.memory | quote }} - cpu: {{ .Values.pod.resources.rgw.requests.cpu | quote }} - limits: - memory: {{ .Values.pod.resources.rgw.limits.memory | quote }} - cpu: {{ .Values.pod.resources.rgw.limits.cpu | quote }} - {{- end }} +{{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} ports: - containerPort: {{ .Values.network.port.rgw_target }} env: + - name: K8S_HOST_NETWORK + value: "1" - name: RGW_CIVETWEB_PORT value: "{{ .Values.network.port.rgw_target }}" - name: CEPH_DAEMON diff --git a/ceph/templates/job-bootstrap.yaml b/ceph/templates/job-bootstrap.yaml index 6aae2af70f..95094fbdc1 100644 --- a/ceph/templates/job-bootstrap.yaml +++ b/ceph/templates/job-bootstrap.yaml @@ -25,7 +25,7 @@ spec: spec: restartPolicy: OnFailure nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.bootstrap.node_selector_key }}: {{ .Values.labels.bootstrap.node_selector_value }} initContainers: {{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: diff --git a/ceph/templates/service.yaml b/ceph/templates/service-mon.yaml similarity index 92% rename from ceph/templates/service.yaml rename to ceph/templates/service-mon.yaml index e0202e557e..6746c4dcf4 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service-mon.yaml @@ -13,14 +13,12 @@ # limitations under the License. {{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} --- kind: Service apiVersion: v1 metadata: name: {{ .Values.endpoints.ceph_mon.hosts.default }} - labels: - app: ceph - daemon: mon annotations: # In kubernetes 1.6 and beyond, it seems there was a change in behavior # requiring us to tolerate unready endpoints to form a quorum. I can only @@ -35,7 +33,6 @@ spec: protocol: TCP targetPort: {{ .Values.network.port.mon }} selector: - app: ceph - daemon: mon +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} clusterIP: None {{- end }} diff --git a/ceph/templates/service-rgw.yaml b/ceph/templates/service-rgw.yaml index c93e82de3d..6f270042bb 100644 --- a/ceph/templates/service-rgw.yaml +++ b/ceph/templates/service-rgw.yaml @@ -14,21 +14,18 @@ {{- if .Values.manifests_enabled.deployment }} {{- if .Values.ceph.enabled.rgw }} +{{- $envAll := . }} --- apiVersion: v1 kind: Service metadata: name: ceph-rgw - labels: - app: ceph - daemon: rgw spec: ports: - port: {{ .Values.network.port.rgw_ingress }} protocol: TCP targetPort: {{ .Values.network.port.rgw_target }} selector: - app: ceph - daemon: rgw +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{- end }} {{- end }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storageclass.yaml similarity index 100% rename from ceph/templates/storage.yaml rename to ceph/templates/storageclass.yaml diff --git a/ceph/values.yaml b/ceph/values.yaml index c8630e6b8e..eac45b5512 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -18,7 +18,6 @@ manifests_enabled: deployment: true replicas: - mon: 3 rgw: 3 mon_check: 1 @@ -34,8 +33,21 @@ images: pull_policy: Always labels: - node_selector_key: ceph-storage - node_selector_value: enabled + mon: + node_selector_key: ceph-mon + node_selector_value: enabled + mds: + node_selector_key: ceph-mds + node_selector_value: enabled + osd: + node_selector_key: ceph-osd + node_selector_value: enabled + rgw: + node_selector_key: ceph-rgw + node_selector_value: enabled + bootstrap: + node_selector_key: ceph-mon + node_selector_value: enabled pod: lifecycle: @@ -104,8 +116,10 @@ secrets: admin: ceph-client-admin-keyring network: - public: "192.168.0.0/16" - cluster: "192.168.0.0/16" +# public: "192.168.0.0/16" +# cluster: "192.168.0.0/16" + public: "10.1.10.0/24" + cluster: "10.1.10.0/24" port: mon: 6789 rgw_ingress: 80 diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index a31c5da82d..d6684a1f46 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -255,16 +255,25 @@ Node Labels First, we must label our nodes according to their role. Although we are labeling ``all`` nodes, you are free to label only the nodes you wish. You must have at least one, although a minimum of three are recommended. +In the case of Ceph, it is important to note that Ceph monitors +and OSDs are each deployed as a ``DaemonSet``. Be aware that +labeling an even number of monitor nodes can result in trouble +when trying to reach a quorum. + Nodes are labeled according to their Openstack roles: -* **Storage Nodes:** ``ceph-storage`` +* **Ceph MON Nodes:** ``ceph-mon`` +* **Ceph OSD Nodes:** ``ceph-osd`` +* **Ceph MDS Nodes:** ``ceph-mds`` * **Control Plane:** ``openstack-control-plane`` * **Compute Nodes:** ``openvswitch``, ``openstack-compute-node`` :: kubectl label nodes openstack-control-plane=enabled --all - kubectl label nodes ceph-storage=enabled --all + kubectl label nodes ceph-mon=enabled --all + kubectl label nodes ceph-osd=enabled --all + kubectl label nodes ceph-mds=enabled --all kubectl label nodes openvswitch=enabled --all kubectl label nodes openstack-compute-node=enabled --all @@ -281,15 +290,17 @@ Download the latest copy of Openstack-Helm: Ceph Preparation and Installation --------------------------------- -Ceph must be aware of the OSD cluster and public networks. These CIDR -ranges are the exact same ranges you used earlier in your Calico -deployment yaml. Export this variable to your deployment environment by -issuing the following commands: +Ceph takes advantage of host networking. For Ceph to be aware of the +OSD cluster and public networks, you must set the CIDR ranges to be the +subnet range that your host machines are running on. In the example provided, +the host's subnet CIDR is ``10.26.0.0/26``, but you will need to replace this +to reflect your cluster. Export these variables to your deployment environment +by issuing the following commands: :: - export osd_cluster_network=192.168.0.0/16 - export osd_public_network=192.168.0.0/16 + export osd_cluster_network=10.26.0.0/26 + export osd_public_network=10.26.0.0/26 Helm Preparation ---------------- diff --git a/mariadb/values.yaml b/mariadb/values.yaml index ef821ae035..a473326c38 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -38,7 +38,7 @@ images: volume: enabled: true class_name: general - size: 2Gi + size: 5Gi labels: node_selector_key: openstack-control-plane diff --git a/tools/gate/basic_launch.sh b/tools/gate/basic_launch.sh index 4b4dbc7f98..6c3613eeac 100755 --- a/tools/gate/basic_launch.sh +++ b/tools/gate/basic_launch.sh @@ -15,6 +15,7 @@ set -ex : ${WORK_DIR:="$(pwd)"} source ${WORK_DIR}/tools/gate/funcs/helm.sh source ${WORK_DIR}/tools/gate/funcs/kube.sh +source ${WORK_DIR}/tools/gate/funcs/network.sh helm_build @@ -33,7 +34,9 @@ helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper kube_wait_for_pods openstack 180 if [ "x$PVC_BACKEND" == "xceph" ]; then - kubectl label nodes ceph-storage=enabled --all + kubectl label nodes ceph-mon=enabled --all + kubectl label nodes ceph-osd=enabled --all + kubectl label nodes ceph-mds=enabled --all CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <