AIAP multinode deployment

It deploys 3 controlplane and 2 worker nodes using AIAP gate. + kubectl --context target-cluster get node NAME STATUS ROLES AGE VERSION node01 Ready control-plane,master 48m v1.21.2 node03 Ready worker 43s v1.21.2 node04 Ready control-plane,master 16m v1.21.2 node05 Ready control-plane,master 26m v1.21.2 node06 Ready worker 54s v1.21.2 Tested with 32 GB node. https://zuul.opendev.org/t/openstack/build/da1c2c440d3d4026ab454de73ee518e2/logs Closes: #652 Closes: #228 Change-Id: Ie2267e15ed75b57e2e27f45b9be19ddcf7b0a0a8
2021-11-05 06:58:52 -07:00 · 2021-11-05 06:58:52 -07:00 · 13c4e82f7e
commit 13c4e82f7e
parent 4dfe4afa34
18 changed files with 117 additions and 26 deletions
--- a/manifests/function/k8scontrol/controlplane.yaml
+++ b/manifests/function/k8scontrol/controlplane.yaml
@ -3,7 +3,7 @@ apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
 metadata:
  name: cluster-controlplane
 spec:
-  replicas: 1
+  replicas: 3
  version: v1.21.2
  machineTemplate:
    infrastructureRef:
--- a/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh
+++ b/manifests/function/phase-helpers/check_ingress_ctrl/kubectl_check_ingress_ctrl.sh
@ -18,12 +18,11 @@ export TIMEOUT=${TIMEOUT:-60}

 end=$(($(date +%s) + $TIMEOUT))
 while true; do
-  export TARGET_IP="$(kubectl --request-timeout 10s \
-    --context $KCTL_CONTEXT \
-    --namespace ingress \
-    get pods \
-    -l app.kubernetes.io/component=controller \
-    -o jsonpath='{.items[*].status.hostIP}')"
+  export TARGET_IP="$(kubectl -n kube-system get po | \
+	  grep -i apiserver | \
+	  cut -f1 -d ' ' | \
+	  xargs kubectl -n kube-system \
+	  get po $1 -o=jsonpath='{.items[0].status.hostIP}')"
  if [ ! -z $TARGET_IP ]; then
    break
  else
--- a/manifests/function/workers-capm3/machinedeployment.yaml
+++ b/manifests/function/workers-capm3/machinedeployment.yaml
@ -6,7 +6,7 @@ metadata:
    cluster.x-k8s.io/cluster-name: target-cluster
 spec:
  clusterName: target-cluster
-  replicas: 1
+  replicas: 2
  selector:
    matchLabels:
      cluster.x-k8s.io/cluster-name: target-cluster
--- a/manifests/phases/executors.yaml
+++ b/manifests/phases/executors.yaml
@ -34,6 +34,7 @@ config:
        value: "provisioned"
  pruneOptions:
    prune: false
+  inventoryPolicy: force-adopt
 ---
 apiVersion: airshipit.org/v1alpha1
 kind: KubernetesApply
--- a/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml
+++ b/manifests/site/test-site/ephemeral/controlplane/kustomization.yaml
@ -11,3 +11,14 @@ namespace: target-infra

 transformers:
  - ../../../../type/gating/ephemeral/controlplane/replacements
+
+patchesJson6902:
+  - target:
+      group: controlplane.cluster.x-k8s.io
+      version: v1alpha4
+      kind: KubeadmControlPlane
+      name: cluster-controlplane
+    path: patch_json6902_controlplane.yaml
+
+patchesStrategicMerge:
+  - versions-airshipctl.yaml
--- a/manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml
+++ b/manifests/site/test-site/ephemeral/controlplane/patch_json6902_controlplane.yaml
@ -0,0 +1,3 @@
+- op: replace
+  path: "/spec/replicas"
+  value: 1
--- a/manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml
+++ b/manifests/site/test-site/ephemeral/controlplane/versions-airshipctl.yaml
@ -0,0 +1,12 @@
+# Override default controlplane image location
+apiVersion: airshipit.org/v1alpha1
+kind: VersionsCatalogue
+metadata:
+  name: versions-airshipctl
+spec:
+  files:
+    k8scontrol:
+      # Host the image in a locally served location for CI
+      cluster_controlplane_image:
+        url: http://10.23.24.137:80/images/control-plane.qcow2
+        checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum
--- a/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml
+++ b/manifests/site/test-site/host-inventory/hostgenerator/host-generation.yaml
@ -13,3 +13,5 @@ hosts:
    - node02
    - node03
    - node04
+    - node05
+    - node06
--- a/manifests/site/test-site/target/catalogues/shareable/hosts.yaml
+++ b/manifests/site/test-site/target/catalogues/shareable/hosts.yaml
@ -54,15 +54,45 @@ hosts:
        pxe: 52:54:00:b6:ed:23
      hardwareProfile: default # defined in the hardwareprofile-example function
    node04:
-      bootMode: UEFI
-      macAddress: 52:54:00:36:5e:e3
-      bmcAddress: redfish+http://10.23.25.2:8000/redfish/v1/Systems/air-target-2
+      bootMode: legacy
+      macAddress: 52:54:00:01:71:d8
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-2
      bmcUsername: username
      bmcPassword: password
+      disableCertificateVerification: false
      ipAddresses:
        oam-ipv4: 10.23.25.104
        pxe-ipv4: 10.23.24.104
      macAddresses:
-        oam: 52:54:00:dc:ab:04
-        pxe: 52:54:00:51:0b:e4
+        oam: 52:54:00:f0:57:e0
+        pxe: 52:54:00:01:71:d8
+      hardwareProfile: default # defined in the hostgenerator-m3 function
+    node05:
+      bootMode: legacy
+      macAddress: 52:54:00:a3:ee:18
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-target-3
+      bmcUsername: username
+      bmcPassword: password
+      disableCertificateVerification: false
+      ipAddresses:
+        oam-ipv4: 10.23.25.105
+        pxe-ipv4: 10.23.24.105
+      macAddresses:
+        oam: 52:54:00:91:36:49
+        pxe: 52:54:00:a3:ee:18
+      hardwareProfile: default # defined in the hostgenerator-m3 function
+    node06:
+      bootMode: legacy
+      macAddress: 52:54:00:b6:ed:19
+      bmcAddress: redfish+http://10.23.25.1:8000/redfish/v1/Systems/air-worker-2
+      bmcUsername: username
+      bmcPassword: password
+      disableCertificateVerification: false
+      ipAddresses:
+        oam-ipv4: 10.23.25.106
+        pxe-ipv4: 10.23.24.106
+      macAddresses:
+        oam: 52:54:00:9b:27:08
+        pxe: 52:54:00:b6:ed:19
      hardwareProfile: default # defined in the hardwareprofile-example function
+
--- a/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml
+++ b/manifests/site/test-site/target/controlplane/hostgenerator/host-generation.yaml
@ -7,3 +7,5 @@ metadata:
  name: host-generation-catalogue
 hosts:
  m3:
+  - node04
+  - node05
--- a/manifests/site/test-site/target/controlplane/kustomization.yaml
+++ b/manifests/site/test-site/target/controlplane/kustomization.yaml
@ -7,6 +7,9 @@ resources:
  - ../catalogues
  - nodes

+patchesStrategicMerge:
+  - metal3machinetemplate.yaml
+
 namespace: target-infra

 commonLabels:
--- a/manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml
+++ b/manifests/site/test-site/target/controlplane/metal3machinetemplate.yaml
@ -0,0 +1,16 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha5
+kind: Metal3MachineTemplate
+metadata:
+  annotations:
+    config.kubernetes.io/path: metal3machinetemplate_cluster-controlplane.yaml
+  name: cluster-controlplane
+spec:
+  template:
+    spec:
+      hostSelector:
+        matchLabels:
+          airshipit.org/k8s-role: controlplane-host
+      image:
+        url: http://10.23.24.137:80/images/control-plane.qcow2
+        checksum: http://10.23.24.137:80/images/control-plane.qcow2.md5sum
--- a/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml
+++ b/manifests/site/test-site/target/workers/hostgenerator/host-generation.yaml
@ -8,3 +8,4 @@ metadata:
 hosts:
  m3:
    - node03
+    - node06
--- a/manifests/type/gating/phases/plan.yaml
+++ b/manifests/type/gating/phases/plan.yaml
@ -99,6 +99,8 @@ phases:
  - name: eject-virtual-media-ephemeral
  # Power off Ephemeral baremetal host avoid DHCP conflict
  - name: power-off-ephemeral
+  # Create target k8s cluster resources
+  - name: controlplane-target
  # List all nodes in target cluster
  # Scripts for this phase placed in manifests/function/phase-helpers/get_node/
  # To get ConfigMap for this phase, execute `airshipctl phase render --source config -k ConfigMap`
@ -124,6 +126,7 @@ validation:
    - VariableCatalogue
  crdList:
    - airshipctl/manifests/function/airshipctl-schemas
+    - airshipctl/manifests/function/baremetal-operator/v0.5.0/upstream/crd
 ---
 apiVersion: airshipit.org/v1alpha1
 kind: PhasePlan
--- a/playbooks/airship-airshipctl-build-gate.yaml
+++ b/playbooks/airship-airshipctl-build-gate.yaml
@ -69,6 +69,7 @@
        path: "{{ roleinputvar.path }}"
      loop:
        - { name: 'target', path: 'ephemeral/controlplane' }
+        - { name: 'target', path: 'target/controlplane' }
        - { name: 'ephemeral', path: 'ephemeral/bootstrap' }
        - { name: 'worker', path: 'target/workers' }
      loop_control:
@ -96,12 +97,12 @@
            target_vm_cfg: "{{ target_vm_cfg }}"
            target_vm_memory_mb: 7168
            target_vm_vcpus: 2
-            target_vms_count: 1
+            target_vms_count: 3
            worker_disk_size: 30G
            worker_vm_cfg: "{{ worker_vm_cfg }}"
            worker_vm_memory_mb: 7168
            worker_vm_vcpus: 2
-            worker_vms_count: 1
+            worker_vms_count: 2
        airship_gate_file_exchanger:
          servername: "localhost"
          ip:
--- a/playbooks/get-vm-config.yaml
+++ b/playbooks/get-vm-config.yaml
@ -17,7 +17,6 @@
      {{ airship_config_manifest_directory }}/{{ airship_config_site_path }}/{{ path }} 2>/dev/null |
    kustomize cfg grep  "kind=BareMetalHost"
  register: bmh_command
-  failed_when: "bmh_command.stdout == ''"
  environment:
    KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
    SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
@ -37,6 +36,7 @@
    KUSTOMIZE_ENABLE_ALPHA_COMMANDS: "true"
    SOPS_IMPORT_PGP: "{{ airship_config_pgp }}"
  with_items: "{{ bmh }}"
+  when: bmh|length > 0

 - name: get links from network data per BareMetalHost object
  set_fact:
@ -47,6 +47,8 @@
          map(attribute='stringData.networkData') | map('from_yaml') |
          map(attribute='links') | list
      }}
+  when: bmh|length > 0
+

 - name: define list of VM mac addresses and VM boot mode
  set_fact:
@ -64,6 +66,12 @@
  loop_control:
    index_var: idx
  register: vm_cfg_fact
+  when: bmh|length > 0
+
+- set_fact:
+    "{{ name }}_vm_cfg": "{{  (lookup('vars', name + '_vm_cfg') |list)  + (vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list) }}"
+  when: "{{ name + '_vm_cfg' }} is defined and bmh|length > 0"

 - set_fact:
    "{{ name }}_vm_cfg": "{{ vm_cfg_fact.results | map(attribute='ansible_facts.vm_cfg') | list }}"
+  when: "{{ name + '_vm_cfg' }} is not defined and bmh|length > 0"
--- a/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml
+++ b/tools/airship-in-a-pod/examples/base/airship-in-a-pod.yaml
@ -24,7 +24,6 @@ data:
  AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_USERNAME: ""
  AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_HTTP_PASSWORD: ""
  AIRSHIP_CONFIG_MANIFEST_REPO_AUTH_SSH_PASSWORD: ""
-  SOPS_IMPORT_PGP: ""
 ---

 apiVersion: v1
--- a/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml
+++ b/tools/airship-in-a-pod/infra-builder/assets/opt/ansible/playbooks/roles/build-infra/defaults/main.yaml
@ -18,7 +18,7 @@ floorplan:

 ephemeral_node:
  name: air-ephemeral
-  cpu: 4
+  cpu: 2
  ram: 6144
  nat_mac_address: [ "52:54:00:9b:27:02" ]
  prov_mac_address: [ "52:54:00:b6:ed:02" ]
@ -26,22 +26,22 @@ ephemeral_node:
    - 30G

 target_nodes:
-  count: 1
+  count: 3
  name: air-target
  cpu: 2
-  ram: 7168
-  nat_mac_address: [ "52:54:00:9b:27:4c" ]
-  prov_mac_address: [ "52:54:00:b6:ed:31" ]
+  ram: 6144
+  nat_mac_address: [ "52:54:00:9b:27:4c", "52:54:00:f0:57:e0", "52:54:00:91:36:49" ]
+  prov_mac_address: [ "52:54:00:b6:ed:31", "52:54:00:01:71:d8", "52:54:00:a3:ee:18" ]
  block:
    - 30G

 worker_nodes:
-  count: 1
+  count: 2
  name: air-worker
  cpu: 2
-  ram: 7168
-  nat_mac_address: [ "52:54:00:9b:27:07" ]
-  prov_mac_address: [ "52:54:00:b6:ed:23" ]
+  ram: 6144
+  nat_mac_address: [ "52:54:00:9b:27:07", "52:54:00:9b:27:08" ]
+  prov_mac_address: [ "52:54:00:b6:ed:23", "52:54:00:b6:ed:19"  ]
  block:
    - 30G