diff --git a/manifests/function/phase-helpers/kustomization.yaml b/manifests/function/phase-helpers/kustomization.yaml index cc20e86e4..d4007eeff 100644 --- a/manifests/function/phase-helpers/kustomization.yaml +++ b/manifests/function/phase-helpers/kustomization.yaml @@ -6,3 +6,4 @@ resources: - get_node - wait_pods - pause_bmh +- wait_cluster diff --git a/manifests/function/phase-helpers/wait_cluster/kubectl_wait_cluster.sh b/manifests/function/phase-helpers/wait_cluster/kubectl_wait_cluster.sh new file mode 100644 index 000000000..8833d2f93 --- /dev/null +++ b/manifests/function/phase-helpers/wait_cluster/kubectl_wait_cluster.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +export TIMEOUT=${TIMEOUT:-3600} + +end=$(($(date +%s) + $TIMEOUT)) +echo "Waiting $TIMEOUT seconds for cluster to reach controlPlaneReady condition" 1>&2 +while true; do + # TODO(vkuzmin): Add ability to wait for multiple clusters + if [ "$(kubectl \ + --request-timeout 20s \ + --context $KCTL_CONTEXT \ + get -f $RENDERED_BUNDLE_PATH \ + -o jsonpath='{.status.controlPlaneReady}')" == "true" ] + then + echo "Getting information about cluster" 1>&2 + kubectl \ + --request-timeout 20s \ + --context $KCTL_CONTEXT \ + get -f $RENDERED_BUNDLE_PATH 1>&2 + echo "Cluster successfully reach controlPlaneReady condition" 1>&2 + break + else + now=$(date +%s) + if [ $now -gt $end ]; then + echo "Cluster didn't reach controlPlaneReady condition before TIMEOUT=$TIMEOUT, exiting" 1>&2 + exit 1 + fi + sleep 15 + fi +done diff --git a/manifests/function/phase-helpers/wait_cluster/kustomization.yaml b/manifests/function/phase-helpers/wait_cluster/kustomization.yaml new file mode 100644 index 000000000..c81d04b9f --- /dev/null +++ b/manifests/function/phase-helpers/wait_cluster/kustomization.yaml @@ -0,0 +1,6 @@ +configMapGenerator: +- name: kubectl-wait-cluster + options: + disableNameSuffixHash: true + files: + - script=kubectl_wait_cluster.sh diff --git a/manifests/function/phase-helpers/wait_pods/kubectl_wait_pods.sh b/manifests/function/phase-helpers/wait_pods/kubectl_wait_pods.sh index 07004826a..4385ce595 100644 --- a/manifests/function/phase-helpers/wait_pods/kubectl_wait_pods.sh +++ b/manifests/function/phase-helpers/wait_pods/kubectl_wait_pods.sh @@ -14,4 +14,13 @@ set -xe -kubectl --kubeconfig $KUBECONFIG --context $KCTL_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=600s 1>&2 +export TIMEOUT=${TIMEOUT:-3000s} + +echo "Waiting for all pods in the cluster with context in kubeconfg ${KCTL_CONTEXT} to reach ready condition" 1>&2 +kubectl --context $KCTL_CONTEXT \ + wait \ + --all-namespaces \ + --for=condition=Ready \ + pods \ + --all \ + --timeout=$TIMEOUT 1>&2 diff --git a/manifests/phases/executors.yaml b/manifests/phases/executors.yaml index c45ec1e98..73f14ffea 100644 --- a/manifests/phases/executors.yaml +++ b/manifests/phases/executors.yaml @@ -410,3 +410,19 @@ configRef: kind: ConfigMap name: kubectl-pause-bmh apiVersion: v1 +--- +apiVersion: airshipit.org/v1alpha1 +kind: GenericContainer +metadata: + name: kubectl-wait-cluster +spec: + image: quay.io/airshipit/toolbox:latest + hostNetwork: true + envVars: + - RESOURCE_GROUP_FILTER=cluster.x-k8s.io + - RESOURCE_VERSION_FILTER=v1alpha3 + - RESOURCE_KIND_FILTER=Cluster +configRef: + kind: ConfigMap + name: kubectl-wait-cluster + apiVersion: v1 diff --git a/manifests/phases/phases.yaml b/manifests/phases/phases.yaml index 72b253849..e3a5410e8 100644 --- a/manifests/phases/phases.yaml +++ b/manifests/phases/phases.yaml @@ -378,3 +378,15 @@ config: kind: GenericContainer name: kubectl-pause-bmh documentEntryPoint: ephemeral/controlplane +--- +apiVersion: airshipit.org/v1alpha1 +kind: Phase +metadata: + name: kubectl-wait-cluster-target + clusterName: target-cluster +config: + executorRef: + apiVersion: airshipit.org/v1alpha1 + kind: GenericContainer + name: kubectl-wait-cluster + documentEntryPoint: ephemeral/controlplane diff --git a/tools/deployment/33_cluster_move_target_node.sh b/tools/deployment/33_cluster_move_target_node.sh index 4d18e49b8..11943e487 100755 --- a/tools/deployment/33_cluster_move_target_node.sh +++ b/tools/deployment/33_cluster_move_target_node.sh @@ -14,17 +14,6 @@ set -xe -#Default wait timeout is 3600 seconds -export TIMEOUT=${TIMEOUT:-3600} -export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} -export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"} -export CLUSTER_NAMESPACE=${CLUSTER_NAMESPACE:-"default"} -export TARGET_NODE=${TARGET_NODE:-"$(airshipctl phase render controlplane-ephemeral \ - -k BareMetalHost -l airshipit.org/k8s-role=controlplane-host \ - 2> /dev/null | \ - yq .metadata.name | \ - sed 's/"//g')"} - # Annotating BMH objects with a pause label # Scripts for this phase placed in manifests/function/phase-helpers/pause_bmh/ # To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap` @@ -34,27 +23,19 @@ airshipctl phase run kubectl-pause-bmh --debug echo "Move Cluster Object to Target Cluster" airshipctl phase run clusterctl-move -echo "Waiting for pods to be ready" -kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s -kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods --all-namespaces +# Waiting for pods to be ready +# Scripts for this phase placed in manifests/function/phase-helpers/wait_pods/ +# To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap` +# and find ConfigMap with name kubectl-wait-pods +airshipctl phase run kubectl-wait-pods-target --debug + +# Scripts for this phase placed in manifests/function/phase-helpers/get_pods/ +# To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap` +# and find ConfigMap with name kubectl-get-pods +airshipctl phase run kubectl-get-pods-target --debug #Wait till crds are created -end=$(($(date +%s) + $TIMEOUT)) -echo "Waiting $TIMEOUT seconds for crds to be created." -while true; do - if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT -n $CLUSTER_NAMESPACE get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then - echo -e "\nGet CRD status" - kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT -n $CLUSTER_NAMESPACE get bmh - kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT -n $CLUSTER_NAMESPACE get machines - kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT -n $CLUSTER_NAMESPACE get clusters - break - else - now=$(date +%s) - if [ $now -gt $end ]; then - echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT." - exit 1 - fi - echo -n . - sleep 15 - fi -done +# Scripts for this phase placed in manifests/function/phase-helpers/wait_cluster/ +# To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap` +# and find ConfigMap with name kubectl-wait-cluster +airshipctl phase run kubectl-wait-cluster-target --debug