diff --git a/tools/deployment/tenant-ceph/010-relabel-nodes.sh b/tools/deployment/tenant-ceph/010-relabel-nodes.sh new file mode 100755 index 000000000..6c72af887 --- /dev/null +++ b/tools/deployment/tenant-ceph/010-relabel-nodes.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +#NOTE(srwilkers): To simplify the process, we simply remove the ceph related +# labels from half the deployed nodes. This allows us to relabel them with the +# tenant ceph labels instead + +# Find the number of nodes and determine half of them +export NUM_NODES=$(kubectl get nodes -o name | wc -w) +export NUM_RELABEL=$(expr $NUM_NODES / 2) + +# Store all node information in JSON +kubectl get nodes -o json >> /tmp/nodes.json + +# Use jq to find the names of the nodes to relabel by slicing the output at the +# number identified above +export RELABEL_NODES=$(cat /tmp/nodes.json | jq -r '.items[0:(env.NUM_RELABEL|tonumber)] | .[].metadata.name') + +# Relabel the nodes appropriately +for node in $RELABEL_NODES; do + for ceph_label in ceph-mon ceph-osd ceph-mds ceph-rgw ceph-mgr; do + kubectl label node $node $ceph_label-; + kubectl label node $node $ceph_label-tenant=enabled; + done; + kubectl label node $node tenant-ceph-control-plane=enabled; +done; diff --git a/tools/deployment/tenant-ceph/020-ingress.sh b/tools/deployment/tenant-ceph/020-ingress.sh new file mode 100755 index 000000000..7a142dc74 --- /dev/null +++ b/tools/deployment/tenant-ceph/020-ingress.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Lint and package chart +make ingress + +#NOTE: Deploy global ingress +tee /tmp/ingress-kube-system.yaml << EOF +deployment: + mode: cluster + type: DaemonSet +network: + host_namespace: true +EOF +helm upgrade --install ingress-kube-system ./ingress \ + --namespace=kube-system \ + --values=/tmp/ingress-kube-system.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM} + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh kube-system + +#NOTE: Display info +helm status ingress-kube-system + +#NOTE: Deploy namespaced ingress controllers +for NAMESPACE in osh-infra ceph tenant-ceph; do + #NOTE: Deploy namespace ingress + helm upgrade --install ingress-${NAMESPACE} ./ingress \ + --namespace=${NAMESPACE} + + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE} + + #NOTE: Display info + helm status ingress-${NAMESPACE} +done diff --git a/tools/deployment/tenant-ceph/030-ceph.sh b/tools/deployment/tenant-ceph/030-ceph.sh new file mode 100755 index 000000000..c1a2c8297 --- /dev/null +++ b/tools/deployment/tenant-ceph/030-ceph.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Deploy command +[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt +CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xubuntu" ] && \ + [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi +if [ "x${ID}" == "xcentos" ]; then + CRUSH_TUNABLES=hammer +fi +tee /tmp/ceph.yaml << EOF +endpoints: + ceph_mon: + namespace: ceph + port: + mon: + default: 6789 + ceph_mgr: + namespace: ceph + port: + mgr: + default: 7000 + metrics: + default: 9283 +network: + public: ${CEPH_PUBLIC_NETWORK} + cluster: ${CEPH_CLUSTER_NETWORK} +deployment: + storage_secrets: true + ceph: true + rbd_provisioner: true + cephfs_provisioner: true + client_secrets: false + rgw_keystone_user_and_endpoints: false +manifests: + deployment_mds: false +bootstrap: + enabled: true +conf: + ceph: + global: + fsid: ${CEPH_FS_ID} + mon: + mon_clock_drift_allowed: 2.0 + rgw_ks: + enabled: true + pool: + crush: + tunables: ${CRUSH_TUNABLES} + target: + # NOTE(portdirect): 5 nodes, with one osd per node + osd: 3 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/ceph/osd/journal-one +storageclass: + rbd: + ceph_configmap_name: ceph-etc + cephfs: + provision_storage_class: false +ceph_mgr_modules_config: + prometheus: + server_port: 9283 +monitoring: + prometheus: + enabled: true + ceph_mgr: + port: 9283 +EOF + +for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do + helm upgrade --install ${CHART} ./${CHART} \ + --namespace=ceph \ + --values=/tmp/ceph.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY} + + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh ceph 1200 + + #NOTE: Validate deploy + MON_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="application=ceph" \ + --selector="component=mon" \ + --no-headers | awk '{ print $1; exit }') + kubectl exec -n ceph ${MON_POD} -- ceph -s +done diff --git a/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh new file mode 120000 index 000000000..f6c0f5f2e --- /dev/null +++ b/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh @@ -0,0 +1 @@ +../multinode/035-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/tenant-ceph/040-tenant-ceph.sh b/tools/deployment/tenant-ceph/040-tenant-ceph.sh new file mode 100755 index 000000000..a92b7b8e8 --- /dev/null +++ b/tools/deployment/tenant-ceph/040-tenant-ceph.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Deploy command +[ -s /tmp/tenant-ceph-fs-uuid.txt ] || uuidgen > /tmp/tenant-ceph-fs-uuid.txt +CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +TENANT_CEPH_FS_ID="$(cat /tmp/tenant-ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xubuntu" ] && \ + [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi +if [ "x${ID}" == "xcentos" ]; then + CRUSH_TUNABLES=hammer +fi +tee /tmp/tenant-ceph.yaml << EOF +endpoints: + identity: + namespace: openstack + object_store: + namespace: openstack + ceph_mon: + namespace: tenant-ceph + port: + mon: + default: 6790 + ceph_mgr: + namespace: tenant-ceph + port: + mgr: + default: 7001 + metrics: + default: 9284 +network: + public: ${CEPH_PUBLIC_NETWORK} + cluster: ${CEPH_CLUSTER_NETWORK} +deployment: + storage_secrets: true + ceph: true + rbd_provisioner: false + cephfs_provisioner: false + client_secrets: false + rgw_keystone_user_and_endpoints: false +labels: + mon: + node_selector_key: ceph-mon-tenant + osd: + node_selector_key: ceph-osd-tenant + rgw: + node_selector_key: ceph-rgw-tenant + mgr: + node_selector_key: ceph-mgr-tenant + job: + node_selector_key: tenant-ceph-control-plane +storageclass: + rbd: + ceph_configmap_name: tenant-ceph-etc + provision_storage_class: false + name: tenant-rbd + admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass + admin_secret_namespace: tenant-ceph + user_secret_name: pvc-tenant-ceph-client-key + cephfs: + provision_storage_class: false + name: cephfs + user_secret_name: pvc-tenant-ceph-cephfs-client-key + admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass + admin_secret_namespace: tenant-ceph +bootstrap: + enabled: true +manifests: + deployment_mds: false +ceph_mgr_modules_config: + prometheus: + server_port: 9284 +monitoring: + prometheus: + enabled: true + ceph_mgr: + port: 9284 +conf: + ceph: + global: + fsid: ${TENANT_CEPH_FS_ID} + rgw_ks: + enabled: true + pool: + crush: + tunables: ${CRUSH_TUNABLES} + target: + osd: 2 + pg_per_osd: 100 + storage: + osd: + - data: + type: directory + location: /var/lib/openstack-helm/tenant-ceph/osd/osd-one + journal: + type: directory + location: /var/lib/openstack-helm/tenant-ceph/osd/journal-one + mon: + directory: /var/lib/openstack-helm/tenant-ceph/mon +EOF + +for CHART in ceph-mon ceph-osd ceph-client; do + helm upgrade --install tenant-${CHART} ./${CHART} \ + --namespace=tenant-ceph \ + --values=/tmp/tenant-ceph.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY} + + #NOTE: Wait for deploy + ./tools/deployment/common/wait-for-pods.sh tenant-ceph 1200 + + #NOTE: Validate deploy + MON_POD=$(kubectl get pods \ + --namespace=tenant-ceph \ + --selector="application=ceph" \ + --selector="component=mon" \ + --no-headers | awk '{ print $1; exit }') + kubectl exec -n tenant-ceph ${MON_POD} -- ceph -s +done diff --git a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh new file mode 100755 index 000000000..cf087d100 --- /dev/null +++ b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Deploy command +CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" +tee /tmp/tenant-ceph-openstack-config.yaml <