Add tenant ceph deployment as nonvoting check

This adds a nonvoting check that will deploy two ceph clusters
and then deploy two radosgw instances, each one backed by a unique
ceph cluster. This allows us validate whether we can reliably
deploy multiple ceph clusters, as in the case of tenant-ceph outlined
in openstack-helm specs

Change-Id: I76a416eddcdb3ea2416d475ea8c8756925cd9580
Co-Authored-By: Meghan Heisler <mh783g@att.com>

tools/deployment/tenant-ceph/010-relabel-nodes.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#NOTE(srwilkers): To simplify the process, we simply remove the ceph related
+# labels from half the deployed nodes. This allows us to relabel them with the
+# tenant ceph labels instead
+
+# Find the number of nodes and determine half of them
+export NUM_NODES=$(kubectl get nodes -o name | wc -w)
+export NUM_RELABEL=$(expr $NUM_NODES / 2)
+
+# Store all node information in JSON
+kubectl get nodes -o json >> /tmp/nodes.json
+
+# Use jq to find the names of the nodes to relabel by slicing the output at the
+# number identified above
+export RELABEL_NODES=$(cat /tmp/nodes.json | jq -r '.items[0:(env.NUM_RELABEL|tonumber)] | .[].metadata.name')
+
+# Relabel the nodes appropriately
+for node in $RELABEL_NODES; do
+  for ceph_label in ceph-mon ceph-osd ceph-mds ceph-rgw ceph-mgr; do
+    kubectl label node $node $ceph_label-;
+    kubectl label node $node $ceph_label-tenant=enabled;
+  done;
+  kubectl label node $node tenant-ceph-control-plane=enabled;
+done;

tools/deployment/tenant-ceph/020-ingress.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make ingress
+
+#NOTE: Deploy global ingress
+tee /tmp/ingress-kube-system.yaml << EOF
+deployment:
+  mode: cluster
+  type: DaemonSet
+network:
+  host_namespace: true
+EOF
+helm upgrade --install ingress-kube-system ./ingress \
+  --namespace=kube-system \
+  --values=/tmp/ingress-kube-system.yaml \
+  ${OSH_INFRA_EXTRA_HELM_ARGS} \
+  ${OSH_INFRA_EXTRA_HELM_ARGS_INGRESS_KUBE_SYSTEM}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh kube-system
+
+#NOTE: Display info
+helm status ingress-kube-system
+
+#NOTE: Deploy namespaced ingress controllers
+for NAMESPACE in osh-infra ceph tenant-ceph; do
+  #NOTE: Deploy namespace ingress
+  helm upgrade --install ingress-${NAMESPACE} ./ingress \
+    --namespace=${NAMESPACE}
+
+  #NOTE: Wait for deploy
+  ./tools/deployment/common/wait-for-pods.sh ${NAMESPACE}
+
+  #NOTE: Display info
+  helm status ingress-${NAMESPACE}
+done

tools/deployment/tenant-ceph/030-ceph.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Deploy command
+[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt
+CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)"
+#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this
+# should be set to 'hammer'
+. /etc/os-release
+if [ "x${ID}" == "xubuntu" ] && \
+   [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then
+  CRUSH_TUNABLES=hammer
+else
+  CRUSH_TUNABLES=null
+fi
+if [ "x${ID}" == "xcentos" ]; then
+  CRUSH_TUNABLES=hammer
+fi
+tee /tmp/ceph.yaml << EOF
+endpoints:
+  ceph_mon:
+    namespace: ceph
+    port:
+      mon:
+        default: 6789
+  ceph_mgr:
+    namespace: ceph
+    port:
+      mgr:
+        default: 7000
+      metrics:
+        default: 9283
+network:
+  public: ${CEPH_PUBLIC_NETWORK}
+  cluster: ${CEPH_CLUSTER_NETWORK}
+deployment:
+  storage_secrets: true
+  ceph: true
+  rbd_provisioner: true
+  cephfs_provisioner: true
+  client_secrets: false
+  rgw_keystone_user_and_endpoints: false
+manifests:
+  deployment_mds: false
+bootstrap:
+  enabled: true
+conf:
+  ceph:
+    global:
+      fsid: ${CEPH_FS_ID}
+    mon:
+      mon_clock_drift_allowed: 2.0
+  rgw_ks:
+    enabled: true
+  pool:
+    crush:
+      tunables: ${CRUSH_TUNABLES}
+    target:
+      # NOTE(portdirect): 5 nodes, with one osd per node
+      osd: 3
+      pg_per_osd: 100
+  storage:
+    osd:
+      - data:
+          type: directory
+          location: /var/lib/openstack-helm/ceph/osd/osd-one
+        journal:
+          type: directory
+          location: /var/lib/openstack-helm/ceph/osd/journal-one
+storageclass:
+  rbd:
+    ceph_configmap_name: ceph-etc
+  cephfs:
+    provision_storage_class: false
+ceph_mgr_modules_config:
+  prometheus:
+    server_port: 9283
+monitoring:
+  prometheus:
+    enabled: true
+    ceph_mgr:
+      port: 9283
+EOF
+
+for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
+  helm upgrade --install ${CHART} ./${CHART} \
+    --namespace=ceph \
+    --values=/tmp/ceph.yaml \
+    ${OSH_INFRA_EXTRA_HELM_ARGS} \
+    ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY}
+
+  #NOTE: Wait for deploy
+  ./tools/deployment/common/wait-for-pods.sh ceph 1200
+
+  #NOTE: Validate deploy
+  MON_POD=$(kubectl get pods \
+    --namespace=ceph \
+    --selector="application=ceph" \
+    --selector="component=mon" \
+    --no-headers | awk '{ print $1; exit }')
+  kubectl exec -n ceph ${MON_POD} -- ceph -s
+done

tools/deployment/tenant-ceph/035-ceph-ns-activate.sh

@@ -0,0 +1 @@
+../multinode/035-ceph-ns-activate.sh

tools/deployment/tenant-ceph/040-tenant-ceph.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Deploy command
+[ -s /tmp/tenant-ceph-fs-uuid.txt ] || uuidgen > /tmp/tenant-ceph-fs-uuid.txt
+CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+TENANT_CEPH_FS_ID="$(cat /tmp/tenant-ceph-fs-uuid.txt)"
+#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this
+# should be set to 'hammer'
+. /etc/os-release
+if [ "x${ID}" == "xubuntu" ] && \
+   [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then
+  CRUSH_TUNABLES=hammer
+else
+  CRUSH_TUNABLES=null
+fi
+if [ "x${ID}" == "xcentos" ]; then
+  CRUSH_TUNABLES=hammer
+fi
+tee /tmp/tenant-ceph.yaml << EOF
+endpoints:
+  identity:
+    namespace: openstack
+  object_store:
+    namespace: openstack
+  ceph_mon:
+    namespace: tenant-ceph
+    port:
+      mon:
+        default: 6790
+  ceph_mgr:
+    namespace: tenant-ceph
+    port:
+      mgr:
+        default: 7001
+      metrics:
+        default: 9284
+network:
+  public: ${CEPH_PUBLIC_NETWORK}
+  cluster: ${CEPH_CLUSTER_NETWORK}
+deployment:
+  storage_secrets: true
+  ceph: true
+  rbd_provisioner: false
+  cephfs_provisioner: false
+  client_secrets: false
+  rgw_keystone_user_and_endpoints: false
+labels:
+  mon:
+    node_selector_key: ceph-mon-tenant
+  osd:
+    node_selector_key: ceph-osd-tenant
+  rgw:
+    node_selector_key: ceph-rgw-tenant
+  mgr:
+    node_selector_key: ceph-mgr-tenant
+  job:
+    node_selector_key: tenant-ceph-control-plane
+storageclass:
+  rbd:
+    ceph_configmap_name: tenant-ceph-etc
+    provision_storage_class: false
+    name: tenant-rbd
+    admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass
+    admin_secret_namespace: tenant-ceph
+    user_secret_name: pvc-tenant-ceph-client-key
+  cephfs:
+    provision_storage_class: false
+    name: cephfs
+    user_secret_name: pvc-tenant-ceph-cephfs-client-key
+    admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass
+    admin_secret_namespace: tenant-ceph
+bootstrap:
+  enabled: true
+manifests:
+  deployment_mds: false
+ceph_mgr_modules_config:
+  prometheus:
+    server_port: 9284
+monitoring:
+  prometheus:
+    enabled: true
+    ceph_mgr:
+      port: 9284
+conf:
+  ceph:
+    global:
+      fsid: ${TENANT_CEPH_FS_ID}
+  rgw_ks:
+    enabled: true
+  pool:
+    crush:
+      tunables: ${CRUSH_TUNABLES}
+    target:
+      osd: 2
+      pg_per_osd: 100
+  storage:
+    osd:
+      - data:
+          type: directory
+          location: /var/lib/openstack-helm/tenant-ceph/osd/osd-one
+        journal:
+          type: directory
+          location: /var/lib/openstack-helm/tenant-ceph/osd/journal-one
+    mon:
+      directory: /var/lib/openstack-helm/tenant-ceph/mon
+EOF
+
+for CHART in ceph-mon ceph-osd ceph-client; do
+  helm upgrade --install tenant-${CHART} ./${CHART} \
+    --namespace=tenant-ceph \
+    --values=/tmp/tenant-ceph.yaml \
+    ${OSH_INFRA_EXTRA_HELM_ARGS} \
+    ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY}
+
+  #NOTE: Wait for deploy
+  ./tools/deployment/common/wait-for-pods.sh tenant-ceph 1200
+
+  #NOTE: Validate deploy
+  MON_POD=$(kubectl get pods \
+    --namespace=tenant-ceph \
+    --selector="application=ceph" \
+    --selector="component=mon" \
+    --no-headers | awk '{ print $1; exit }')
+  kubectl exec -n tenant-ceph ${MON_POD} -- ceph -s
+done

tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Deploy command
+CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+tee /tmp/tenant-ceph-openstack-config.yaml <<EOF
+endpoints:
+  identity:
+    namespace: openstack
+  object_store:
+    namespace: openstack
+  ceph_mon:
+    namespace: tenant-ceph
+    port:
+      mon:
+        default: 6790
+network:
+  public: ${CEPH_PUBLIC_NETWORK}
+  cluster: ${CEPH_CLUSTER_NETWORK}
+deployment:
+  storage_secrets: false
+  ceph: false
+  rbd_provisioner: false
+  cephfs_provisioner: false
+  client_secrets: true
+  rgw_keystone_user_and_endpoints: false
+bootstrap:
+  enabled: false
+conf:
+  rgw_ks:
+    enabled: true
+storageclass:
+  rbd:
+    ceph_configmap_name: tenant-ceph-etc
+    provision_storage_class: false
+    name: tenant-rbd
+    admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass
+    admin_secret_namespace: tenant-ceph
+    user_secret_name: pvc-tenant-ceph-client-key
+  cephfs:
+    provision_storage_class: false
+    name: cephfs
+    admin_secret_name: pvc-tenant-ceph-conf-combined-storageclass
+    admin_secret_namespace: tenant-ceph
+    user_secret_name: pvc-tenant-ceph-cephfs-client-key
+EOF
+helm upgrade --install tenant-ceph-openstack-config ./ceph-provisioners \
+  --namespace=openstack \
+  --values=/tmp/tenant-ceph-openstack-config.yaml \
+  ${OSH_EXTRA_HELM_ARGS} \
+  ${OSH_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE}
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh openstack
+
+#NOTE: Validate Deployment info
+helm status tenant-ceph-openstack-config

tools/deployment/tenant-ceph/050-radosgw-osh-infra.sh

@@ -0,0 +1 @@
+../multinode/115-radosgw-osh-infra.sh

tools/deployment/tenant-ceph/060-radosgw-openstack.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+#NOTE: Lint and package chart
+make ceph-rgw
+
+#NOTE: Deploy command
+CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)"
+tee /tmp/radosgw-openstack.yaml <<EOF
+endpoints:
+  identity:
+    namespace: openstack
+  object_store:
+    namespace: openstack
+  ceph_mon:
+    namespace: tenant-ceph
+    port:
+      mon:
+        default: 6790
+network:
+  public: ${CEPH_PUBLIC_NETWORK}
+  cluster: ${CEPH_CLUSTER_NETWORK}
+deployment:
+  storage_secrets: false
+  ceph: true
+  rbd_provisioner: false
+  cephfs_provisioner: false
+  client_secrets: false
+  rgw_keystone_user_and_endpoints: false
+bootstrap:
+  enabled: false
+conf:
+  rgw_ks:
+    enabled: false
+  rgw_s3:
+    enabled: true
+secrets:
+  keyrings:
+    admin: pvc-tenant-ceph-client-key
+    rgw: os-ceph-bootstrap-rgw-keyring
+  identity:
+    admin: ceph-keystone-admin
+    swift: ceph-keystone-user
+    user_rgw: ceph-keystone-user-rgw
+ceph_client:
+  configmap: tenant-ceph-etc
+EOF
+helm upgrade --install radosgw-openstack ./ceph-rgw \
+  --namespace=openstack \
+  --values=/tmp/radosgw-openstack.yaml
+
+#NOTE: Wait for deploy
+./tools/deployment/common/wait-for-pods.sh openstack
+
+#NOTE: Validate Deployment info
+helm status radosgw-openstack
+
+helm test radosgw-openstack

zuul.d/jobs.yaml

@@ -62,6 +62,28 @@
         - ./tools/deployment/multinode/130-fluent-logging.sh
         - ./tools/deployment/multinode/140-kibana.sh
 
+- job:
+    name: openstack-helm-infra-tenant-ceph
+    parent: openstack-helm-infra-functional
+    nodeset: openstack-helm-five-node-ubuntu
+    timeout: 7200
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+      - playbooks/osh-infra-deploy-docker.yaml
+      - playbooks/osh-infra-build.yaml
+      - playbooks/osh-infra-deploy-k8s.yaml
+    run: playbooks/osh-infra-gate-runner.yaml
+    post-run: playbooks/osh-infra-collect-logs.yaml
+    vars:
+      gate_scripts:
+        - ./tools/deployment/tenant-ceph/010-relabel-nodes.sh
+        - ./tools/deployment/tenant-ceph/020-ingress.sh
+        - ./tools/deployment/tenant-ceph/030-ceph.sh
+        - ./tools/deployment/tenant-ceph/035-ceph-ns-activate.sh
+        - ./tools/deployment/tenant-ceph/040-tenant-ceph.sh
+        - ./tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh
+        - ./tools/deployment/tenant-ceph/050-radosgw-osh-infra.sh
+        - ./tools/deployment/tenant-ceph/060-radosgw-openstack.sh
 
 - job:
     name: openstack-helm-infra-ubuntu

zuul.d/project.yaml

@@ -32,6 +32,8 @@
         # override functionality
         - openstack-helm-infra-airship-divingbell:
             voting: false
+        - openstack-helm-infra-tenant-ceph:
+            voting: false
     gate:
       jobs:
         - openstack-helm-lint