From 79d75267eaac8699022e3a9113ca10f648a861a8 Mon Sep 17 00:00:00 2001 From: Gage Hugo Date: Mon, 25 Oct 2021 14:15:49 -0500 Subject: [PATCH] Move osh-infra jobs to use helm3 This change updates many of the deployment scripts to properly handle deploying each service via helm 3 and updates each job to use the helm v3 install script. Change-Id: I90a7b59231376b9179439c2554e46449d59b9c15 --- tools/deployment/apparmor/005-deploy-k8s.sh | 2 +- tools/deployment/apparmor/030-mariadb.sh | 2 +- .../deployment/apparmor/090-elasticsearch.sh | 2 +- tools/deployment/apparmor/100-fluentbit.sh | 2 +- .../apparmor/110-fluentd-daemonset.sh | 2 +- .../apparmor/140-ceph-radosgateway.sh | 2 +- tools/deployment/common/005-deploy-k8s.sh | 72 +------------------ .../common/010-deploy-docker-registry.sh | 16 ++++- tools/deployment/common/metacontroller.sh | 12 ++++ .../elastic-beats/005-deploy-k8s.sh | 2 +- .../federated-monitoring/005-deploy-k8s.sh | 2 +- .../federated-monitoring/060-prometheus.sh | 2 +- .../070-federated-prometheus.sh | 2 +- .../federated-monitoring/090-grafana.sh | 2 +- tools/deployment/keystone-auth/060-mariadb.sh | 2 +- tools/deployment/multinode/030-ceph.sh | 4 +- .../multinode/035-ceph-ns-activate.sh | 2 +- tools/deployment/multinode/045-mariadb.sh | 2 +- tools/deployment/multinode/050-prometheus.sh | 2 +- tools/deployment/multinode/100-grafana.sh | 2 +- .../multinode/115-radosgw-osh-infra.sh | 2 +- .../deployment/multinode/120-elasticsearch.sh | 2 +- .../network-policy/005-deploy-k8s.sh | 2 +- .../deployment/network-policy/045-mariadb.sh | 2 +- .../openstack-support/005-deploy-k8s.sh | 2 +- .../openstack-support/025-ceph-ns-activate.sh | 2 +- .../openstack-support/030-rabbitmq.sh | 2 +- .../100-ceph-radosgateway.sh | 2 +- .../openstack-support/130-cinder.sh | 2 +- .../osh-infra-local-storage/005-deploy-k8s.sh | 2 +- .../osh-infra-local-storage/040-prometheus.sh | 2 +- .../osh-infra-logging-tls/005-deploy-k8s.sh | 2 +- .../osh-infra-logging-tls/020-ceph.sh | 4 +- .../025-ceph-ns-activate.sh | 2 +- .../030-radosgw-osh-infra.sh | 2 +- .../050-elasticsearch.sh | 2 +- .../osh-infra-logging/005-deploy-k8s.sh | 2 +- .../deployment/osh-infra-logging/020-ceph.sh | 4 +- .../osh-infra-logging/025-ceph-ns-activate.sh | 2 +- .../030-radosgw-osh-infra.sh | 2 +- .../osh-infra-logging/050-elasticsearch.sh | 2 +- .../005-deploy-k8s.sh | 2 +- .../030-nfs-provisioner.sh | 12 ++++ .../osh-infra-monitoring-tls/045-mariadb.sh | 2 +- .../050-prometheus.sh | 2 +- .../osh-infra-monitoring-tls/110-grafana.sh | 2 +- .../osh-infra-monitoring-tls/120-nagios.sh | 2 +- .../osh-infra-monitoring/005-deploy-k8s.sh | 2 +- .../030-nfs-provisioner.sh | 12 ++++ .../osh-infra-monitoring/045-mariadb.sh | 2 +- .../osh-infra-monitoring/050-prometheus.sh | 2 +- .../osh-infra-monitoring/110-grafana.sh | 2 +- .../osh-infra-monitoring/120-nagios.sh | 2 +- .../podsecuritypolicy/005-deploy-k8s.sh | 2 +- tools/deployment/tenant-ceph/030-ceph.sh | 4 +- .../deployment/tenant-ceph/040-tenant-ceph.sh | 4 +- .../045-tenant-ceph-ns-activate.sh | 2 +- .../tenant-ceph/060-radosgw-openstack.sh | 2 +- tools/gate/deploy-k8s.sh | 17 ++++- 59 files changed, 125 insertions(+), 132 deletions(-) mode change 100755 => 120000 tools/deployment/common/005-deploy-k8s.sh diff --git a/tools/deployment/apparmor/005-deploy-k8s.sh b/tools/deployment/apparmor/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/apparmor/005-deploy-k8s.sh +++ b/tools/deployment/apparmor/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/030-mariadb.sh b/tools/deployment/apparmor/030-mariadb.sh index 297e49ef3..b53fb698d 100755 --- a/tools/deployment/apparmor/030-mariadb.sh +++ b/tools/deployment/apparmor/030-mariadb.sh @@ -33,4 +33,4 @@ helm upgrade --install mariadb ./mariadb \ # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace osh-infra diff --git a/tools/deployment/apparmor/090-elasticsearch.sh b/tools/deployment/apparmor/090-elasticsearch.sh index 94c3d50e1..c3ffeb992 100755 --- a/tools/deployment/apparmor/090-elasticsearch.sh +++ b/tools/deployment/apparmor/090-elasticsearch.sh @@ -76,4 +76,4 @@ helm upgrade --install elasticsearch ./elasticsearch \ # Delete the test pod if it still exists kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -helm test elasticsearch +helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/apparmor/100-fluentbit.sh b/tools/deployment/apparmor/100-fluentbit.sh index 347fff5c7..dca71cc07 100755 --- a/tools/deployment/apparmor/100-fluentbit.sh +++ b/tools/deployment/apparmor/100-fluentbit.sh @@ -34,4 +34,4 @@ helm upgrade --install fluentbit ./fluentbit \ # Delete the test pod if it still exists kubectl delete pods -l application=fluentbit,release_group=fluentbit,component=test --namespace=osh-infra --ignore-not-found -helm test fluentbit +helm test fluentbit --namespace osh-infra diff --git a/tools/deployment/apparmor/110-fluentd-daemonset.sh b/tools/deployment/apparmor/110-fluentd-daemonset.sh index c9bc79a55..d8cfea6ed 100755 --- a/tools/deployment/apparmor/110-fluentd-daemonset.sh +++ b/tools/deployment/apparmor/110-fluentd-daemonset.sh @@ -169,4 +169,4 @@ helm upgrade --install fluentd-daemonset ./fluentd \ # Delete the test pod if it still exists kubectl delete pods -l application=fluentd,release_group=fluentd-daemonset,component=test --namespace=osh-infra --ignore-not-found -helm test fluentd-daemonset +helm test fluentd-daemonset --namespace osh-infra diff --git a/tools/deployment/apparmor/140-ceph-radosgateway.sh b/tools/deployment/apparmor/140-ceph-radosgateway.sh index 13602babe..f0f82cc0e 100755 --- a/tools/deployment/apparmor/140-ceph-radosgateway.sh +++ b/tools/deployment/apparmor/140-ceph-radosgateway.sh @@ -62,4 +62,4 @@ openstack endpoint list # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=radosgw-openstack,component=rgw-test --namespace=openstack --ignore-not-found -helm test radosgw-openstack --timeout 900 +helm test radosgw-openstack --namespace openstack --timeout 900s diff --git a/tools/deployment/common/005-deploy-k8s.sh b/tools/deployment/common/005-deploy-k8s.sh deleted file mode 100755 index 14d40b907..000000000 --- a/tools/deployment/common/005-deploy-k8s.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -MINIKUBE_AIO_DEFAULT="docker.io/openstackhelm/minikube-aio:latest-ubuntu_bionic" -: ${MINIKUBE_AIO:=${MINIKUBE_AIO_DEFAULT}} - -export DEBCONF_NONINTERACTIVE_SEEN=true -export DEBIAN_FRONTEND=noninteractive - -echo "DefaultLimitMEMLOCK=16384" | sudo tee -a /etc/systemd/system.conf -sudo systemctl daemon-reexec - -# Install required packages for K8s on host -wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - -RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}') -sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/ -${RELEASE_NAME} main" -sudo -E apt-get update -sudo -E apt-get install -y \ - docker.io - -# Starting to pull early in parallel -sudo -E docker pull -q ${MINIKUBE_AIO} & - -sudo -E apt-get install -y \ - socat \ - jq \ - util-linux \ - ceph-common \ - rbd-nbd \ - nfs-common \ - bridge-utils \ - conntrack \ - iptables - -sudo -E tee /etc/modprobe.d/rbd.conf << EOF -install rbd /bin/true -EOF - -set +x; -# give 2 minutes to pull the image (usually takes less than 30-60s) and proceed. If something bad -# happens we'll see it on 'docker create' -echo "Waiting for ${MINIKUBE_AIO} image is pulled" -i=0 -while [ "$i" -le "60" ]; do - (( ++i )) - sudo docker inspect ${MINIKUBE_AIO} && break || sleep 2; -done &> /dev/null; set -x -TMP_DIR=$(mktemp -d) -sudo docker create --name minikube-aio ${MINIKUBE_AIO} bash -sudo docker export minikube-aio | tar x -C ${TMP_DIR} -sudo docker rm minikube-aio -sudo docker rmi ${MINIKUBE_AIO} -${TMP_DIR}/install.sh -rm ${TMP_DIR} -rf - -make diff --git a/tools/deployment/common/005-deploy-k8s.sh b/tools/deployment/common/005-deploy-k8s.sh new file mode 120000 index 000000000..003bfbb8e --- /dev/null +++ b/tools/deployment/common/005-deploy-k8s.sh @@ -0,0 +1 @@ +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/common/010-deploy-docker-registry.sh b/tools/deployment/common/010-deploy-docker-registry.sh index 6073c5a67..08f4d7185 100755 --- a/tools/deployment/common/010-deploy-docker-registry.sh +++ b/tools/deployment/common/010-deploy-docker-registry.sh @@ -19,6 +19,20 @@ make nfs-provisioner make redis make registry +for NAMESPACE in docker-nfs docker-registry; do +tee /tmp/${NAMESPACE}-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${NAMESPACE} + name: ${NAMESPACE} + name: ${NAMESPACE} +EOF + +kubectl create -f /tmp/${NAMESPACE}-ns.yaml +done + #NOTE: Deploy nfs for the docker registry tee /tmp/docker-registry-nfs-provisioner.yaml << EOF labels: @@ -55,4 +69,4 @@ helm upgrade --install docker-registry ./registry \ # Delete the test pod if it still exists kubectl delete pods -l application=redis,release_group=docker-registry-redis,component=test --namespace=docker-registry --ignore-not-found #NOTE: Run helm tests -helm test docker-registry-redis +helm test docker-registry-redis --namespace docker-registry diff --git a/tools/deployment/common/metacontroller.sh b/tools/deployment/common/metacontroller.sh index 81e84d351..954ac3a25 100755 --- a/tools/deployment/common/metacontroller.sh +++ b/tools/deployment/common/metacontroller.sh @@ -26,6 +26,18 @@ if [ -z "$crds" ]; then echo "No crd exists of APIGroup metacontroller.k8s.io" fi +tee /tmp/${namespace}-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${namespace} + name: ${namespace} + name: ${namespace} +EOF + +kubectl create -f /tmp/${namespace}-ns.yaml + #NOTE: Deploy command helm upgrade --install metacontroller ./metacontroller \ --namespace=$namespace \ diff --git a/tools/deployment/elastic-beats/005-deploy-k8s.sh b/tools/deployment/elastic-beats/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/elastic-beats/005-deploy-k8s.sh +++ b/tools/deployment/elastic-beats/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/005-deploy-k8s.sh b/tools/deployment/federated-monitoring/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/federated-monitoring/005-deploy-k8s.sh +++ b/tools/deployment/federated-monitoring/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/060-prometheus.sh b/tools/deployment/federated-monitoring/060-prometheus.sh index 96763c810..e05668346 100755 --- a/tools/deployment/federated-monitoring/060-prometheus.sh +++ b/tools/deployment/federated-monitoring/060-prometheus.sh @@ -61,5 +61,5 @@ for release in prometheus-one prometheus-two prometheus-three; do # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus-$release,component=test --namespace=osh-infra --ignore-not-found - helm test prometheus-$release + helm test prometheus-$release --namespace osh-infra done diff --git a/tools/deployment/federated-monitoring/070-federated-prometheus.sh b/tools/deployment/federated-monitoring/070-federated-prometheus.sh index 89f10995f..b1c8591ac 100755 --- a/tools/deployment/federated-monitoring/070-federated-prometheus.sh +++ b/tools/deployment/federated-monitoring/070-federated-prometheus.sh @@ -60,4 +60,4 @@ helm upgrade --install federated-prometheus ./prometheus \ # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=federated-prometheus,component=test --namespace=osh-infra --ignore-not-found -helm test federated-prometheus +helm test federated-prometheus --namespace osh-infra diff --git a/tools/deployment/federated-monitoring/090-grafana.sh b/tools/deployment/federated-monitoring/090-grafana.sh index 92e6ca703..cfe61666f 100755 --- a/tools/deployment/federated-monitoring/090-grafana.sh +++ b/tools/deployment/federated-monitoring/090-grafana.sh @@ -159,7 +159,7 @@ helm upgrade --install grafana ./grafana \ # Delete the test pod if it still exists kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found -helm test grafana +helm test grafana --namespace osh-infra echo "Get list of all configured datasources in Grafana" curl -u admin:password http://grafana.osh-infra.svc.cluster.local/api/datasources | jq -r . diff --git a/tools/deployment/keystone-auth/060-mariadb.sh b/tools/deployment/keystone-auth/060-mariadb.sh index 89714bd30..9187c56c3 100755 --- a/tools/deployment/keystone-auth/060-mariadb.sh +++ b/tools/deployment/keystone-auth/060-mariadb.sh @@ -32,4 +32,4 @@ helm upgrade --install mariadb ./mariadb \ # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=openstack --ignore-not-found #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace openstack diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh index d41a54ae6..07bbf6938 100755 --- a/tools/deployment/multinode/030-ceph.sh +++ b/tools/deployment/multinode/030-ceph.sh @@ -129,8 +129,8 @@ done # Delete the test pod if it still exists kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --timeout 900 +helm test ceph-osd --namespace ceph --timeout 900s # Delete the test pod if it still exists kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --timeout 900 +helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/multinode/035-ceph-ns-activate.sh b/tools/deployment/multinode/035-ceph-ns-activate.sh index 85ed568ae..292abfb5f 100755 --- a/tools/deployment/multinode/035-ceph-ns-activate.sh +++ b/tools/deployment/multinode/035-ceph-ns-activate.sh @@ -55,4 +55,4 @@ helm upgrade --install ceph-osh-infra-config ./ceph-provisioners \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=ceph-osh-infra-config,component=provisioner-test --namespace=osh-infra --ignore-not-found -helm test ceph-osh-infra-config --timeout 600 +helm test ceph-osh-infra-config --namespace osh-infra --timeout 600s diff --git a/tools/deployment/multinode/045-mariadb.sh b/tools/deployment/multinode/045-mariadb.sh index 54ed14bcb..e04f0501b 100755 --- a/tools/deployment/multinode/045-mariadb.sh +++ b/tools/deployment/multinode/045-mariadb.sh @@ -33,4 +33,4 @@ helm upgrade --install mariadb ./mariadb \ # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace osh-infra diff --git a/tools/deployment/multinode/050-prometheus.sh b/tools/deployment/multinode/050-prometheus.sh index 4592dc298..d92dc97ab 100755 --- a/tools/deployment/multinode/050-prometheus.sh +++ b/tools/deployment/multinode/050-prometheus.sh @@ -33,4 +33,4 @@ helm upgrade --install prometheus ./prometheus \ # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found #NOTE: Run helm tests -helm test prometheus +helm test prometheus --namespace osh-infra diff --git a/tools/deployment/multinode/100-grafana.sh b/tools/deployment/multinode/100-grafana.sh index 451471616..06c96700f 100755 --- a/tools/deployment/multinode/100-grafana.sh +++ b/tools/deployment/multinode/100-grafana.sh @@ -33,4 +33,4 @@ helm upgrade --install grafana ./grafana \ # Delete the test pod if it still exists kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found #NOTE: Run helm tests -helm test grafana +helm test grafana --namespace osh-infra diff --git a/tools/deployment/multinode/115-radosgw-osh-infra.sh b/tools/deployment/multinode/115-radosgw-osh-infra.sh index c72c805af..f90de80bc 100755 --- a/tools/deployment/multinode/115-radosgw-osh-infra.sh +++ b/tools/deployment/multinode/115-radosgw-osh-infra.sh @@ -69,4 +69,4 @@ helm upgrade --install radosgw-osh-infra ./ceph-rgw \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=radosgw-osh-infra,component=rgw-test --namespace=osh-infra --ignore-not-found -helm test radosgw-osh-infra --timeout 900 +helm test radosgw-osh-infra --namespace osh-infra --timeout 900s diff --git a/tools/deployment/multinode/120-elasticsearch.sh b/tools/deployment/multinode/120-elasticsearch.sh index 363c3809d..f572c380a 100755 --- a/tools/deployment/multinode/120-elasticsearch.sh +++ b/tools/deployment/multinode/120-elasticsearch.sh @@ -72,4 +72,4 @@ helm upgrade --install elasticsearch ./elasticsearch \ # Delete the test pod if it still exists kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found #NOTE: Run helm tests -helm test elasticsearch +helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/network-policy/005-deploy-k8s.sh b/tools/deployment/network-policy/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/network-policy/005-deploy-k8s.sh +++ b/tools/deployment/network-policy/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/network-policy/045-mariadb.sh b/tools/deployment/network-policy/045-mariadb.sh index cba8b09b2..09eca4ff8 100755 --- a/tools/deployment/network-policy/045-mariadb.sh +++ b/tools/deployment/network-policy/045-mariadb.sh @@ -40,4 +40,4 @@ helm upgrade --install mariadb ./mariadb \ ./tools/deployment/common/wait-for-pods.sh osh-infra #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace osh-infra diff --git a/tools/deployment/openstack-support/005-deploy-k8s.sh b/tools/deployment/openstack-support/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/openstack-support/005-deploy-k8s.sh +++ b/tools/deployment/openstack-support/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/openstack-support/025-ceph-ns-activate.sh b/tools/deployment/openstack-support/025-ceph-ns-activate.sh index 6d976cc3f..e9e205710 100755 --- a/tools/deployment/openstack-support/025-ceph-ns-activate.sh +++ b/tools/deployment/openstack-support/025-ceph-ns-activate.sh @@ -52,7 +52,7 @@ helm upgrade --install ceph-openstack-config ./ceph-provisioners \ #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh openstack -helm test ceph-openstack-config --timeout 600 +helm test ceph-openstack-config --namespace openstack --timeout 600s #NOTE: Validate Deployment info kubectl get -n openstack jobs diff --git a/tools/deployment/openstack-support/030-rabbitmq.sh b/tools/deployment/openstack-support/030-rabbitmq.sh index 0f7f163bc..830f6311e 100755 --- a/tools/deployment/openstack-support/030-rabbitmq.sh +++ b/tools/deployment/openstack-support/030-rabbitmq.sh @@ -31,4 +31,4 @@ helm upgrade --install rabbitmq ./rabbitmq \ #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh openstack -helm test rabbitmq +helm test rabbitmq --namespace openstack diff --git a/tools/deployment/openstack-support/100-ceph-radosgateway.sh b/tools/deployment/openstack-support/100-ceph-radosgateway.sh index 7f9776b32..7511b49a8 100755 --- a/tools/deployment/openstack-support/100-ceph-radosgateway.sh +++ b/tools/deployment/openstack-support/100-ceph-radosgateway.sh @@ -59,4 +59,4 @@ sleep 60 #NOTE(portdirect): Wait for ingress controller to update rules and rest openstack service list openstack endpoint list -helm test radosgw-openstack --timeout 900 +helm test radosgw-openstack --namespace openstack --timeout 900s diff --git a/tools/deployment/openstack-support/130-cinder.sh b/tools/deployment/openstack-support/130-cinder.sh index 87fc00771..7f1d6ef55 100755 --- a/tools/deployment/openstack-support/130-cinder.sh +++ b/tools/deployment/openstack-support/130-cinder.sh @@ -59,4 +59,4 @@ sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and rest openstack volume type list kubectl delete pods -l application=cinder,release_group=cinder,component=test --namespace=openstack --ignore-not-found -helm test cinder --timeout 900 +helm test cinder --namespace openstack --timeout 900s diff --git a/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh b/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh +++ b/tools/deployment/osh-infra-local-storage/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-local-storage/040-prometheus.sh b/tools/deployment/osh-infra-local-storage/040-prometheus.sh index 27ef5b83d..caf52624e 100755 --- a/tools/deployment/osh-infra-local-storage/040-prometheus.sh +++ b/tools/deployment/osh-infra-local-storage/040-prometheus.sh @@ -31,4 +31,4 @@ helm upgrade --install prometheus ./prometheus \ # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found -helm test prometheus +helm test prometheus --namespace osh-infra diff --git a/tools/deployment/osh-infra-logging-tls/005-deploy-k8s.sh b/tools/deployment/osh-infra-logging-tls/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/osh-infra-logging-tls/005-deploy-k8s.sh +++ b/tools/deployment/osh-infra-logging-tls/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging-tls/020-ceph.sh b/tools/deployment/osh-infra-logging-tls/020-ceph.sh index b52f708f6..6d782a166 100755 --- a/tools/deployment/osh-infra-logging-tls/020-ceph.sh +++ b/tools/deployment/osh-infra-logging-tls/020-ceph.sh @@ -222,7 +222,7 @@ done # Delete the test pod if it still exists kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --timeout 900 +helm test ceph-osd --namespace ceph --timeout 900s # Delete the test pod if it still exists kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --timeout 900 +helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh index 1e9e18e12..3068780e0 100755 --- a/tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh +++ b/tools/deployment/osh-infra-logging-tls/025-ceph-ns-activate.sh @@ -54,7 +54,7 @@ helm upgrade --install ceph-osh-infra-config ./ceph-provisioners \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=ceph-osh-infra-config,component=provisioner-test --namespace=osh-infra --ignore-not-found -helm test ceph-osh-infra-config --timeout 600 +helm test ceph-osh-infra-config --namespace osh-infra --timeout 600s #NOTE: Validate Deployment info kubectl get -n osh-infra jobs diff --git a/tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh b/tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh index 6e2b4973b..268a4e34c 100755 --- a/tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh +++ b/tools/deployment/osh-infra-logging-tls/030-radosgw-osh-infra.sh @@ -64,4 +64,4 @@ helm upgrade --install radosgw-osh-infra ./ceph-rgw \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=radosgw-osh-infra,component=rgw-test --namespace=osh-infra --ignore-not-found #NOTE: Test Deployment -helm test radosgw-osh-infra --timeout 900 +helm test radosgw-osh-infra --namespace osh-infra --timeout 900s diff --git a/tools/deployment/osh-infra-logging-tls/050-elasticsearch.sh b/tools/deployment/osh-infra-logging-tls/050-elasticsearch.sh index f1fb337a8..7415f7261 100755 --- a/tools/deployment/osh-infra-logging-tls/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging-tls/050-elasticsearch.sh @@ -116,4 +116,4 @@ helm upgrade --install elasticsearch ./elasticsearch \ # Delete the test pod if it still exists kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -helm test elasticsearch +helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/osh-infra-logging/005-deploy-k8s.sh b/tools/deployment/osh-infra-logging/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/osh-infra-logging/005-deploy-k8s.sh +++ b/tools/deployment/osh-infra-logging/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/020-ceph.sh b/tools/deployment/osh-infra-logging/020-ceph.sh index b52f708f6..6d782a166 100755 --- a/tools/deployment/osh-infra-logging/020-ceph.sh +++ b/tools/deployment/osh-infra-logging/020-ceph.sh @@ -222,7 +222,7 @@ done # Delete the test pod if it still exists kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --timeout 900 +helm test ceph-osd --namespace ceph --timeout 900s # Delete the test pod if it still exists kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --timeout 900 +helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh index 1e9e18e12..3068780e0 100755 --- a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh +++ b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh @@ -54,7 +54,7 @@ helm upgrade --install ceph-osh-infra-config ./ceph-provisioners \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=ceph-osh-infra-config,component=provisioner-test --namespace=osh-infra --ignore-not-found -helm test ceph-osh-infra-config --timeout 600 +helm test ceph-osh-infra-config --namespace osh-infra --timeout 600s #NOTE: Validate Deployment info kubectl get -n osh-infra jobs diff --git a/tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh b/tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh index 2e6c03441..31eae83a8 100755 --- a/tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh +++ b/tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh @@ -71,7 +71,7 @@ helm upgrade --install radosgw-osh-infra ./ceph-rgw \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=radosgw-osh-infra,component=rgw-test --namespace=osh-infra --ignore-not-found #NOTE: Test Deployment -helm test radosgw-osh-infra --timeout 900 +helm test radosgw-osh-infra --namespace osh-infra --timeout 900s #NOTE: RGW needs to be restarted for placement-targets to become accessible kubectl delete pods -l application=ceph,component=rgw -n osh-infra diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index 04901a227..a0755faf3 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -111,4 +111,4 @@ helm upgrade --install elasticsearch ./elasticsearch \ # Delete the test pod if it still exists kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -helm test elasticsearch +helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring-tls/005-deploy-k8s.sh b/tools/deployment/osh-infra-monitoring-tls/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/osh-infra-monitoring-tls/005-deploy-k8s.sh +++ b/tools/deployment/osh-infra-monitoring-tls/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-monitoring-tls/030-nfs-provisioner.sh b/tools/deployment/osh-infra-monitoring-tls/030-nfs-provisioner.sh index 4ca67dd9f..8e0f532a8 100755 --- a/tools/deployment/osh-infra-monitoring-tls/030-nfs-provisioner.sh +++ b/tools/deployment/osh-infra-monitoring-tls/030-nfs-provisioner.sh @@ -16,6 +16,18 @@ set -xe make nfs-provisioner +tee /tmp/nfs-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: nfs + name: nfs + name: nfs +EOF + +kubectl create -f /tmp/nfs-ns.yaml + #NOTE: Deploy nfs instance for logging, monitoring and alerting components tee /tmp/nfs-provisioner.yaml << EOF labels: diff --git a/tools/deployment/osh-infra-monitoring-tls/045-mariadb.sh b/tools/deployment/osh-infra-monitoring-tls/045-mariadb.sh index 28e9bbd93..99aae116c 100755 --- a/tools/deployment/osh-infra-monitoring-tls/045-mariadb.sh +++ b/tools/deployment/osh-infra-monitoring-tls/045-mariadb.sh @@ -35,4 +35,4 @@ helm upgrade --install mariadb ./mariadb \ # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring-tls/050-prometheus.sh b/tools/deployment/osh-infra-monitoring-tls/050-prometheus.sh index 92e1fb2f4..2d7e7f815 100755 --- a/tools/deployment/osh-infra-monitoring-tls/050-prometheus.sh +++ b/tools/deployment/osh-infra-monitoring-tls/050-prometheus.sh @@ -31,4 +31,4 @@ helm upgrade --install prometheus ./prometheus \ # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found -helm test prometheus +helm test prometheus --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring-tls/110-grafana.sh b/tools/deployment/osh-infra-monitoring-tls/110-grafana.sh index 64011f63c..cf37d6c74 100755 --- a/tools/deployment/osh-infra-monitoring-tls/110-grafana.sh +++ b/tools/deployment/osh-infra-monitoring-tls/110-grafana.sh @@ -31,4 +31,4 @@ helm upgrade --install grafana ./grafana \ # Delete the test pod if it still exists kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found -helm test grafana +helm test grafana --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh b/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh index 02343a2d0..a41de6a54 100755 --- a/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh +++ b/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh @@ -30,4 +30,4 @@ helm upgrade --install nagios ./nagios \ # Delete the test pod if it still exists kubectl delete pods -l application=nagios,release_group=nagios,component=test --namespace=osh-infra --ignore-not-found -helm test nagios +helm test nagios --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh b/tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh +++ b/tools/deployment/osh-infra-monitoring/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh b/tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh index 4ca67dd9f..8e0f532a8 100755 --- a/tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh +++ b/tools/deployment/osh-infra-monitoring/030-nfs-provisioner.sh @@ -16,6 +16,18 @@ set -xe make nfs-provisioner +tee /tmp/nfs-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: nfs + name: nfs + name: nfs +EOF + +kubectl create -f /tmp/nfs-ns.yaml + #NOTE: Deploy nfs instance for logging, monitoring and alerting components tee /tmp/nfs-provisioner.yaml << EOF labels: diff --git a/tools/deployment/osh-infra-monitoring/045-mariadb.sh b/tools/deployment/osh-infra-monitoring/045-mariadb.sh index 2966a9cfb..2a0e08d04 100755 --- a/tools/deployment/osh-infra-monitoring/045-mariadb.sh +++ b/tools/deployment/osh-infra-monitoring/045-mariadb.sh @@ -33,4 +33,4 @@ helm upgrade --install mariadb ./mariadb \ # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found #NOTE: Validate the deployment -helm test mariadb +helm test mariadb --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring/050-prometheus.sh b/tools/deployment/osh-infra-monitoring/050-prometheus.sh index bf32bcd89..5685a771d 100755 --- a/tools/deployment/osh-infra-monitoring/050-prometheus.sh +++ b/tools/deployment/osh-infra-monitoring/050-prometheus.sh @@ -31,4 +31,4 @@ helm upgrade --install prometheus ./prometheus \ # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found -helm test prometheus +helm test prometheus --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring/110-grafana.sh b/tools/deployment/osh-infra-monitoring/110-grafana.sh index 19fa9a487..54556391b 100755 --- a/tools/deployment/osh-infra-monitoring/110-grafana.sh +++ b/tools/deployment/osh-infra-monitoring/110-grafana.sh @@ -31,4 +31,4 @@ helm upgrade --install grafana ./grafana \ # Delete the test pod if it still exists kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found -helm test grafana +helm test grafana --namespace osh-infra diff --git a/tools/deployment/osh-infra-monitoring/120-nagios.sh b/tools/deployment/osh-infra-monitoring/120-nagios.sh index 02343a2d0..a41de6a54 100755 --- a/tools/deployment/osh-infra-monitoring/120-nagios.sh +++ b/tools/deployment/osh-infra-monitoring/120-nagios.sh @@ -30,4 +30,4 @@ helm upgrade --install nagios ./nagios \ # Delete the test pod if it still exists kubectl delete pods -l application=nagios,release_group=nagios,component=test --namespace=osh-infra --ignore-not-found -helm test nagios +helm test nagios --namespace osh-infra diff --git a/tools/deployment/podsecuritypolicy/005-deploy-k8s.sh b/tools/deployment/podsecuritypolicy/005-deploy-k8s.sh index 257a39f7a..003bfbb8e 120000 --- a/tools/deployment/podsecuritypolicy/005-deploy-k8s.sh +++ b/tools/deployment/podsecuritypolicy/005-deploy-k8s.sh @@ -1 +1 @@ -../common/005-deploy-k8s.sh \ No newline at end of file +../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/tenant-ceph/030-ceph.sh b/tools/deployment/tenant-ceph/030-ceph.sh index 3e7781bfa..ccdced69a 100755 --- a/tools/deployment/tenant-ceph/030-ceph.sh +++ b/tools/deployment/tenant-ceph/030-ceph.sh @@ -145,7 +145,7 @@ done # Delete the test pod if it still exists kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --timeout 900 +helm test ceph-osd --namespace ceph --timeout 900s # Delete the test pod if it still exists kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --timeout 900 +helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/tenant-ceph/040-tenant-ceph.sh b/tools/deployment/tenant-ceph/040-tenant-ceph.sh index 968c9e5a9..3ea85ef95 100755 --- a/tools/deployment/tenant-ceph/040-tenant-ceph.sh +++ b/tools/deployment/tenant-ceph/040-tenant-ceph.sh @@ -174,5 +174,5 @@ for CHART in ceph-mon ceph-osd ceph-client; do kubectl exec -n tenant-ceph ${MON_POD} -- ceph -s done -helm test tenant-ceph-osd --timeout 900 -helm test tenant-ceph-client --timeout 900 +helm test tenant-ceph-osd --namespace tenant-ceph --timeout 900s +helm test tenant-ceph-client --namespace tenant-ceph --timeout 900s diff --git a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh index b853122a2..68732cdb5 100755 --- a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh +++ b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh @@ -83,4 +83,4 @@ helm upgrade --install tenant-ceph-openstack-config ./ceph-provisioners \ #NOTE: Wait for deploy ./tools/deployment/common/wait-for-pods.sh openstack -helm test tenant-ceph-openstack-config --timeout 600 +helm test tenant-ceph-openstack-config --namespace openstack --timeout 600s diff --git a/tools/deployment/tenant-ceph/060-radosgw-openstack.sh b/tools/deployment/tenant-ceph/060-radosgw-openstack.sh index 49925dae0..1a3f087a9 100755 --- a/tools/deployment/tenant-ceph/060-radosgw-openstack.sh +++ b/tools/deployment/tenant-ceph/060-radosgw-openstack.sh @@ -69,4 +69,4 @@ helm upgrade --install radosgw-openstack ./ceph-rgw \ # Delete the test pod if it still exists kubectl delete pods -l application=ceph,release_group=radosgw-openstack,component=rgw-test --namespace=openstack --ignore-not-found -helm test radosgw-openstack --timeout 900 +helm test radosgw-openstack --namespace openstack --timeout 900s diff --git a/tools/gate/deploy-k8s.sh b/tools/gate/deploy-k8s.sh index baf984045..06b54522f 100755 --- a/tools/gate/deploy-k8s.sh +++ b/tools/gate/deploy-k8s.sh @@ -14,7 +14,7 @@ set -ex : "${HELM_VERSION:="v3.6.3"}" -: "${KUBE_VERSION:="v1.21.5"}" +: "${KUBE_VERSION:="v1.19.16"}" : "${MINIKUBE_VERSION:="v1.22.0"}" : "${CALICO_VERSION:="v3.20"}" : "${YQ_VERSION:="v4.6.0"}" @@ -100,6 +100,12 @@ Environment="NO_PROXY=${NO_PROXY}" EOF fi +# Install required packages for K8s on host +wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - +RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}') +sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/ +${RELEASE_NAME} main" + sudo -E apt-get update sudo -E apt-get install -y \ docker-ce \ @@ -116,7 +122,14 @@ sudo -E apt-get install -y \ make \ bc \ git-review \ - notary + notary \ + ceph-common \ + rbd-nbd \ + nfs-common + +sudo -E tee /etc/modprobe.d/rbd.conf << EOF +install rbd /bin/true +EOF # Prepare tmpfs for etcd when running on CI # CI VMs can have slow I/O causing issues for etcd