Enhance target node deployment on gates

* Add script to install clusterctl
  * Add controlplane yamls for target node
  * Add script to perform cluster move
  * Add script to zuul jobs

Closes: #288
Change-Id: Ia6891df9c9b1da333396e76f11332deeb17ab807
Signed-off-by: Sreejith Punnapuzha <sreejith.punnapuzha@outlook.com>

manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/cabpk/v0.3.3/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/cacpk/v0.3.3/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_clusters.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinedeployments.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machines.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/cluster.x-k8s.io_machinesets.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capi/v0.3.3/crd/bases/exp.cluster.x-k8s.io_machinepools.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machines.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/capm3/v0.3.1/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml

@@ -3,6 +3,8 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   annotations:
     controller-gen.kubebuilder.io/version: v0.2.8
   creationTimestamp: null

manifests/function/ephemeral/secret.yaml

@@ -14,6 +14,7 @@ stringData:
       list: |
           root:deploY!K8s
           deployer:deploY!K8s
+      expire: False
     users:
       - default
       - name: deployer

manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml

@@ -1,6 +1,8 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
+  labels:
+    clusterctl.cluster.x-k8s.io: ""
   name: baremetalhosts.metal3.io
 spec:
   additionalPrinterColumns:

manifests/site/test-site/target/controlplane/control-machine-template-patch.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
+kind: Metal3MachineTemplate
+metadata:
+  name: cluster-controlplane
+spec:
+  template:
+    spec:
+      image:
+        url: http://10.23.24.1:8099/target-image.qcow2
+        checksum: http://10.23.24.1:8099/target-image.qcow2.md5sum

manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml

@@ -0,0 +1,11 @@
+# Site-level, phase-specific lists of hosts to generate
+# This is used by the hostgenerator-m3 function to narrow down the site-level
+# host-catalogue to just the hosts needed for a particular phase.
+apiVersion: airshipit.org/v1alpha1
+kind: VariableCatalogue
+metadata:
+  name: host-generation-catalogue
+hosts:
+  m3:
+    - node01
+    - node02

manifests/site/test-site/target/controlplane/hostgenerator/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../../../../function/hostgenerator-m3
+  - ../../../shared/catalogues/
+  - host-generation.yaml
+
+transformers:
+  - ../../../../../function/hostgenerator-m3/replacements

manifests/site/test-site/target/controlplane/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # TODO (dukov) It's recocommended to upload BareMetalHost objects separately
+  # otherwise nodes will hang in 'registering' state for quite a long time
+  - nodes
+  - ../../../../function/k8scontrol
+patchesStrategicMerge:
+  - control-machine-template-patch.yaml
+commonLabels:
+  airshipit.org/stage: initinfra

manifests/site/test-site/target/controlplane/nodes/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+generators:
+  - ../hostgenerator
+
+commonLabels:
+  airshipit.org/k8s-role: controlplane-host

playbooks/airshipctl-gate-runner.yaml

@@ -16,6 +16,7 @@
       set_fact:
         gate_scripts_default:
           - ./tools/deployment/01_install_kubectl.sh
+          - ./tools/deployment/02_install_clusterctl.sh
           - ./tools/deployment/21_systemwide_executable.sh
           - ./tools/deployment/22_test_configs.sh
           - ./tools/deployment/23_pull_documents.sh
@@ -25,6 +26,7 @@
           - ./tools/deployment/30_deploy_controlplane.sh
           - ./tools/deployment/31_deploy_initinfra_target_node.sh
           - ./tools/deployment/32_cluster_init_target_node.sh
+          - ./tools/deployment/33_cluster_move_target_node.sh
 
     - name: "Run gate scripts"
       include_role:

roles/libvirt-install/tasks/main.yaml

@@ -26,6 +26,7 @@
           - dnsmasq
           - ebtables
           - ovmf
+          - jq
           RedHat:
           - edk2-ovmf
           - libguestfs-tools
@@ -33,6 +34,7 @@
           - libvirt-devel
           - libvirt-daemon-kvm
           - qemu-kvm
+          - jq
     package:
       name: "{{ packages[ansible_os_family] }}"
       state: present

tools/deployment/02_install_clusterctl.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+: ${CLUSTERCTL_VERSION:="v0.3.8"}
+
+# Install clusterctl
+sudo -E curl -sSLo /usr/local/bin/clusterctl \
+  https://github.com/kubernetes-sigs/cluster-api/releases/download/${CLUSTERCTL_VERSION}/clusterctl-linux-amd64
+
+sudo -E chmod +x /usr/local/bin/clusterctl

tools/deployment/33_cluster_move_target_node.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+#Default wait timeout is 3600 seconds
+export TIMEOUT=${TIMEOUT:-3600}
+export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"}
+
+echo "Switch context to ephemeral cluster and set manifest"
+airshipctl config use-context dummy_cluster
+airshipctl config set-context dummy_cluster --manifest dummy_manifest
+
+echo "Check Cluster Status"
+kubectl --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady'
+
+echo "Annotate BMH for target node"
+kubectl --kubeconfig $KUBECONFIG annotate bmh node01 baremetalhost.metal3.io/paused=true
+
+echo "Move Cluster Object to Target Cluster"
+clusterctl --v 20 move --kubeconfig $KUBECONFIG --kubeconfig-context dummy_cluster --to-kubeconfig $KUBECONFIG --to-kubeconfig-context  target-cluster-admin@target-cluster
+
+echo "Switch context to target cluster and set manifest"
+airshipctl config use-context target-cluster-admin@target-cluster
+airshipctl config set-context target-cluster-admin@target-cluster --manifest dummy_manifest
+
+echo "Waiting for pods to be ready"
+kubectl --kubeconfig $KUBECONFIG wait --all-namespaces --for=condition=Ready pods --all --timeout=3000s
+kubectl --kubeconfig $KUBECONFIG get pods --all-namespaces
+
+#Wait till crds are created
+end=$(($(date +%s) + $TIMEOUT))
+echo "Waiting $TIMEOUT seconds for crds to be created."
+while true; do
+    if (kubectl --request-timeout 20s --kubeconfig $KUBECONFIG get cluster target-cluster -o json | jq '.status.controlPlaneReady' | grep -q true) ; then
+        echo -e "\nGet CRD status"
+        kubectl --kubeconfig $KUBECONFIG get bmh
+        kubectl --kubeconfig $KUBECONFIG get machines
+        kubectl --kubeconfig $KUBECONFIG get clusters
+        break
+    else
+        now=$(date +%s)
+        if [ $now -gt $end ]; then
+            echo -e "\nCluster move failed and CRDs was not ready before TIMEOUT."
+            exit 1
+        fi
+        echo -n .
+        sleep 15
+    fi
+done

zuul.d/jobs.yaml

@@ -88,6 +88,7 @@
       site_name: test-site
       gate_scripts:
         - ./tools/deployment/01_install_kubectl.sh
+        - ./tools/deployment/02_install_clusterctl.sh
         # 21_systemwide_executable.sh is run in the build-gate pre-run above
         - ./tools/deployment/22_test_configs.sh
         - ./tools/deployment/24_build_ephemeral_iso.sh
@@ -96,6 +97,7 @@
         - ./tools/deployment/30_deploy_controlplane.sh
         - ./tools/deployment/31_deploy_initinfra_target_node.sh
         - ./tools/deployment/32_cluster_init_target_node.sh
+        - ./tools/deployment/33_cluster_move_target_node.sh
       serve_dir: /srv/iso
       serve_port: 8099
     voting: false