Fix CAPD deployment using phase plan

This PS introduces phase plan for CAPD provider * Adds phase to merge kubeconfig * Patches type/gating plan to use capd phase plan * Patches cluster_map and executor to work with "default" namespace * Fixes CAPD zuul failures Relates-To: #564 Relates-To: #580 Relates-To: #587 Change-Id: I5007970c907bc87dccf6dd9fcb052afc1b5c13f7
2021-06-08 13:04:17 +00:00 · 2021-06-08 13:04:17 +00:00 · 441ef0e3da
commit 441ef0e3da
parent 0cb8e0f5f4
20 changed files with 123 additions and 189 deletions
--- a/manifests/function/capd/v0.3.11/data/kustomization.yaml
+++ b/manifests/function/capd/v0.3.11/data/kustomization.yaml
@ -0,0 +1,2 @@
+resources:
+  - metadata.yaml
--- a/manifests/function/capd/v0.3.11/data/metadata.yaml
+++ b/manifests/function/capd/v0.3.11/data/metadata.yaml
@ -0,0 +1,11 @@
+---
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+metadata:
+  name: repository-metadata
+  labels:
+    airshipit.org/deploy-k8s: "false"
+releaseSeries:
+- major: 0
+  minor: 3
+  contract: v1alpha3
--- a/manifests/function/capd/v0.3.11/kustomization.yaml
+++ b/manifests/function/capd/v0.3.11/kustomization.yaml
@ -5,5 +5,6 @@ commonLabels:

 resources:
 - crd
+- data
 - default
 - webhook
--- a/manifests/function/phase-helpers/kustomization.yaml
+++ b/manifests/function/phase-helpers/kustomization.yaml
@ -13,3 +13,4 @@ resources:
 - wait_bmh
 - wait_label_node
 - check_ingress_ctrl
+- merge_kubeconfig
--- a/manifests/function/phase-helpers/merge_kubeconfig/kubectl_merge_kubeconfig.sh
+++ b/manifests/function/phase-helpers/merge_kubeconfig/kubectl_merge_kubeconfig.sh
@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@ -12,16 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-set -xe
+set -ex

-export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
-export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
+KUBECONFIG=${KUBECONFIG}:${TGT_KUBECONFIG} kubectl config view --flatten > /tmp/kubeconfig-trans

-echo "Stop/Delete ephemeral node"
-kind delete cluster --name "ephemeral-cluster"
+cat /tmp/kubeconfig-trans > ${TGT_KUBECONFIG}

-echo "Deploy worker node"
-airshipctl phase run  workers-target --debug
-
-#Wait till node is created
-kubectl wait --for=condition=ready node --all --timeout=1000s --context $KUBECONFIG_TARGET_CONTEXT --kubeconfig $KUBECONFIG -A
--- a/manifests/function/phase-helpers/merge_kubeconfig/kustomization.yaml
+++ b/manifests/function/phase-helpers/merge_kubeconfig/kustomization.yaml
@ -0,0 +1,6 @@
+configMapGenerator:
+- name: merge-kubeconfig
+  options:
+    disableNameSuffixHash: true
+  files:
+  - script=kubectl_merge_kubeconfig.sh
--- a/manifests/phases/executors.yaml
+++ b/manifests/phases/executors.yaml
@ -563,3 +563,25 @@ spec:
  type: krm
  image: localhost/clusterctl:latest
  hostNetwork: true
+---
+apiVersion: airshipit.org/v1alpha1
+kind: GenericContainer
+metadata:
+  name: merge-kubeconfig
+  labels:
+    airshipit.org/deploy-k8s: "false"
+spec:
+  type: krm
+  image: quay.io/airshipit/toolbox:latest
+  hostNetwork: true
+  envVars:
+    - TGT_KUBECONFIG=/tmp-kubeconfig
+  mounts:
+  - type: bind
+    src: ~/.airship/kubeconfig
+    dst: /tmp-kubeconfig
+    rw: true
+configRef:
+  kind: ConfigMap
+  name: merge-kubeconfig
+  apiVersion: v1
--- a/manifests/phases/phases.yaml
+++ b/manifests/phases/phases.yaml
@ -488,3 +488,14 @@ config:
    apiVersion: airshipit.org/v1alpha1
    kind: GenericContainer
    name: kubectl-check-ingress-ctrl
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
+metadata:
+  name: kubectl-merge-kubeconfig
+  clusterName: target-cluster
+config:
+  executorRef:
+    apiVersion: airshipit.org/v1alpha1
+    kind: GenericContainer
+    name: merge-kubeconfig
--- a/manifests/site/docker-test-site/kustomization.yaml
+++ b/manifests/site/docker-test-site/kustomization.yaml
@ -0,0 +1,2 @@
+resources:
+  - metadata.yaml
--- a/manifests/site/docker-test-site/phases/cluster_map_patch.yaml
+++ b/manifests/site/docker-test-site/phases/cluster_map_patch.yaml
@ -0,0 +1,22 @@
+apiVersion: airshipit.org/v1alpha1
+kind: ClusterMap
+metadata:
+  labels:
+    airshipit.org/deploy-k8s: "false"
+  name: main-map
+map:
+  target-cluster:
+    parent: ephemeral-cluster
+    kubeconfigSources:
+      - type: "filesystem"
+        filesystem:
+          path: ~/.airship/kubeconfig
+          contextName: target-cluster
+      - type: "bundle"
+        bundle:
+          contextName: target-cluster
+      - type: "clusterAPI"
+        clusterAPI:
+          clusterNamespacedName:
+            name: target-cluster
+            namespace: default
--- a/manifests/site/docker-test-site/phases/executor_patch.yaml
+++ b/manifests/site/docker-test-site/phases/executor_patch.yaml
@ -0,0 +1,7 @@
+apiVersion: airshipit.org/v1alpha1
+kind: Clusterctl
+metadata:
+  name: clusterctl_move
+move-options:
+  namespace: default
+action: move
--- a/manifests/site/docker-test-site/phases/infrastructure-providers.json
+++ b/manifests/site/docker-test-site/phases/infrastructure-providers.json
@ -1,7 +1,7 @@
 [{
    "op": "replace",
    "path": "/init-options/infrastructure-providers",
-    "value": ["docker:v0.3.11"]
+    "value": "docker:v0.3.11"
 },
 {
    "op": "replace",
@ -9,10 +9,7 @@
    "value": {
        "name": "docker",
        "type": "InfrastructureProvider",
-        "variable-substitution": true,
-        "versions": {
-            "v0.3.11": "airshipctl/manifests/function/capd/v0.3.11"
-        }
+        "url": "airshipctl/manifests/function/capd/v0.3.11"
    }
 }
 ]
--- a/manifests/site/docker-test-site/phases/kustomization.yaml
+++ b/manifests/site/docker-test-site/phases/kustomization.yaml
@ -1,7 +1,6 @@
 resources:
  - ../../../phases
  - ../../../function/airshipctl-base-catalogues
-  - plan.yaml
 patchesJson6902:
  - target:
      group: airshipit.org
@ -9,6 +8,10 @@ patchesJson6902:
      kind: Clusterctl
      name: "clusterctl_init"
    path: infrastructure-providers.json
+patchesStrategicMerge:
+  - plan_patch.yaml
+  - cluster_map_patch.yaml
+  - executor_patch.yaml
 transformers:
  - ../../../function/clusterctl/replacements
  - ../../../phases/replacements
--- a/manifests/site/docker-test-site/phases/plan.yaml
+++ b/manifests/site/docker-test-site/phases/plan.yaml
@ -1,11 +0,0 @@
-apiVersion: airshipit.org/v1alpha1
-kind: PhasePlan
-metadata:
-  name: phasePlan
-phases:
-  - name: clusterctl-init-ephemeral
-  - name: controlplane-ephemeral
-  - name: initinfra-networking-target
-  - name: clusterctl-init-target
-  - name: clusterctl-move
-  - name: workers-target
--- a/manifests/site/docker-test-site/phases/plan_patch.yaml
+++ b/manifests/site/docker-test-site/phases/plan_patch.yaml
@ -0,0 +1,20 @@
+apiVersion: airshipit.org/v1alpha1
+kind: PhasePlan
+metadata:
+  name: deploy-gating
+description: "Phase plan for docker-test-site deployment"
+phases:
+  - name: clusterctl-init-ephemeral
+  - name: kubectl-wait-deploy-ephemeral
+  - name: kubectl-get-pods-ephemeral
+  - name: controlplane-ephemeral
+  - name: kubectl-merge-kubeconfig
+  - name: kubectl-get-node-target
+  - name: kubectl-get-pods-target
+  - name: initinfra-networking-target
+  - name: kubectl-wait-tigera-target
+  - name: kubectl-get-pods-target
+  - name: clusterctl-init-target
+  - name: kubectl-wait-pods-any-ephemeral
+  - name: clusterctl-move
+  - name: workers-target
--- a/tools/deployment/kind/start_kind.sh
+++ b/tools/deployment/kind/start_kind.sh
@ -39,7 +39,9 @@ set -xe
 : ${KUBECONFIG:="${HOME}/.airship/kubeconfig"}
 : ${TIMEOUT:=3600}
 : ${KIND_CONFIG:=""}
+
 export KIND_EXPERIMENTAL_DOCKER_NETWORK=bridge
+export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}

 echo "cluster name: $CLUSTER";

@ -58,5 +60,8 @@ kubectl wait --for=condition=ready node --all --timeout=1000s --kubeconfig $KUBE
 # By default, kind creates context kind-<cluster_name>
 kubectl config set-context ${CLUSTER} --cluster kind-${CLUSTER} --user kind-${CLUSTER} --kubeconfig $KUBECONFIG

+# Add context for target-cluster
+kubectl config set-context ${KUBECONFIG_TARGET_CONTEXT} --user target-cluster-admin --cluster ${KUBECONFIG_TARGET_CONTEXT} --kubeconfig  $KUBECONFIG
+
 echo "This cluster can be deleted via:"
 echo "${KIND} delete cluster --name ${CLUSTER}"
--- a/tools/deployment/provider_common/30_deploy_controlplane.sh
+++ b/tools/deployment/provider_common/30_deploy_controlplane.sh
@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Example Usage
-# ./tools/deployment/provider_common/30_deploy_controlplane.sh
-
-set -xe
-
-export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
-export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
-export KUBECONFIG_EPHEMERAL_CONTEXT=${KUBECONFIG_EPHEMERAL_CONTEXT:-"ephemeral-cluster"}
-
-echo "create control plane"
-airshipctl phase run controlplane-ephemeral --debug --wait-timeout 1000s
-
-airshipctl cluster get-kubeconfig > ~/.airship/kubeconfig-tmp
-
-mv ~/.airship/kubeconfig-tmp "${KUBECONFIG}"
-
-echo "apply cni as a part of initinfra-networking"
-airshipctl phase run initinfra-networking-target --debug
-
-echo "Check nodes status"
-kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBECONFIG_TARGET_CONTEXT}" wait --for=condition=Ready nodes --all --timeout 4000s
-kubectl get nodes --kubeconfig "${KUBECONFIG}" --context "${KUBECONFIG_TARGET_CONTEXT}"
-
-echo "Waiting for  pods to come up"
-kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBECONFIG_TARGET_CONTEXT}" wait --for=condition=ready pods --all --timeout=4000s -A
-kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBECONFIG_TARGET_CONTEXT}" get pods -A
-
-echo "Check machine status"
-kubectl get machines --kubeconfig ${KUBECONFIG} --context "${KUBECONFIG_EPHEMERAL_CONTEXT}"
-
-echo "Get cluster state for target workload cluster "
-kubectl --kubeconfig ${KUBECONFIG} --context "${KUBECONFIG_EPHEMERAL_CONTEXT}" get cluster
--- a/tools/deployment/provider_common/32_cluster_init_target_node.sh
+++ b/tools/deployment/provider_common/32_cluster_init_target_node.sh
@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -xe
-
-# Example Usage
-# KUBECONFIG=/tmp/$KUBECONFIG \
-# ./tools/deployment/provider_common/32_cluster_init_target_node.sh
-
-export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
-export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
-
-# Get control plane node
-CONTROL_PLANE_NODES=( $(kubectl --context $KUBECONFIG_TARGET_CONTEXT --kubeconfig $KUBECONFIG get --no-headers=true nodes \
-| grep cluster-control-plane | awk '{print $1}') )
-
-# Remove noschedule taint to prevent cluster init from timing out
-for i in "${CONTROL_PLANE_NODES}"; do
-  echo untainting node $i
-  kubectl taint node $i node-role.kubernetes.io/master- --context $KUBECONFIG_TARGET_CONTEXT --kubeconfig $KUBECONFIG --request-timeout 10s
-done
-
-echo "Deploy CAPI components to target cluster"
-airshipctl phase run clusterctl-init-target --debug
-
-echo "Waiting for pods to be ready"
-kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=600s
-kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods --all-namespaces
--- a/tools/deployment/provider_common/33_cluster_move_target_node.sh
+++ b/tools/deployment/provider_common/33_cluster_move_target_node.sh
@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -xe
-
-#Default wait timeout is 3600 seconds
-export TIMEOUT=${TIMEOUT:-3600}
-export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
-export KUBECONFIG_TARGET_CONTEXT=${KUBECONFIG_TARGET_CONTEXT:-"target-cluster"}
-export KUBECONFIG_EPHEMERAL_CONTEXT=${KUBECONFIG_EPHEMERAL_CONTEXT:-"ephemeral-cluster"}
-
-echo "Waiting for machines to come up"
-kubectl --kubeconfig ${KUBECONFIG} --context $KUBECONFIG_EPHEMERAL_CONTEXT wait --for=condition=Ready machines --all --timeout 4000s
-
-#add wait condition
-end=$(($(date +%s) + $TIMEOUT))
-echo "Waiting $TIMEOUT seconds for Machine to be Running."
-while true; do
-    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get machines -o json | jq '.items[0].status.phase' | grep -q "Running") ; then
-        echo -e "\nMachine is Running"
-        kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get machines
-        break
-    else
-        now=$(date +%s)
-        if [ $now -gt $end ]; then
-            echo -e "\nMachine is not in Ruuning phase, Move can't be performed."
-            exit 1
-        fi
-        echo -n .
-        sleep 15
-    fi
-done
-
-airshipctl phase run clusterctl-move
-
-echo "Waiting for pods to be ready"
-kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s
-kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get pods --all-namespaces
-
-#Wait till crds are created
-end=$(($(date +%s) + $TIMEOUT))
-echo "Waiting $TIMEOUT seconds for crds to be created."
-while true; do
-    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then
-        echo -e "\nGet CRD status"
-        kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get machines
-        kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_TARGET_CONTEXT get clusters
-        break
-    else
-        now=$(date +%s)
-        if [ $now -gt $end ]; then
-            echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT."
-            exit 1
-        fi
-        echo -n .
-        sleep 15
-    fi
-done
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -187,7 +187,7 @@
    timeout: 3600
    pre-run: playbooks/airship-airshipctl-deploy-docker.yaml
    run: playbooks/airshipctl-gate-runner.yaml
-    nodeset: airship-airshipctl-single-node
+    nodeset: airship-airshipctl-single-16GB-bionic-node
    irrelevant-files: *noncodefiles
    dependencies:
      - name: openstack-tox-docs
@ -211,14 +211,10 @@
        - ./tools/deployment/provider_common/02_install_jq.sh
        - ./tools/deployment/provider_common/03_install_pip.sh
        - ./tools/deployment/provider_common/04_install_yq.sh
-        - CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/deployment/kind/start_kind.sh
        - AIRSHIP_CONFIG_METADATA_PATH=manifests/site/docker-test-site/metadata.yaml SITE=docker-test-site EXTERNAL_KUBECONFIG="true" ./tools/deployment/22_test_configs.sh
+        - CLUSTER=ephemeral-cluster KIND_CONFIG=./tools/deployment/templates/kind-cluster-with-extramounts ./tools/deployment/kind/start_kind.sh
        - ./tools/deployment/23_pull_documents.sh
-        - PROVIDER=default SITE=docker-test-site ./tools/deployment/26_deploy_capi_ephemeral_node.sh
-        - ./tools/deployment/provider_common/30_deploy_controlplane.sh
-        - ./tools/deployment/provider_common/32_cluster_init_target_node.sh
-        - ./tools/deployment/provider_common/33_cluster_move_target_node.sh
-        - ./tools/deployment/provider_common/34_deploy_worker_node.sh
+        - ./tools/deployment/25_deploy_gating.sh
    voting: false
 - job:
    name: airship-airshipctl-docker-kubebench-conformance