Add Kubespray compute-kit job

We would like to make sure OSH charts work
with K8s clusters managed by third party tools
not only with the K8s cluster deployed by
OSH deploy-env role

Depends-On: I4ffc6653070dbbc6f0766b278acf0ebe2b4ae1e1
Depends-On: I92c73606ed9b9161f39ea1971b3a7db7593982ff
Change-Id: If9bd4720bea951a56cfd6d4f80be37bd17bcb149

Makefile

@@ -15,7 +15,7 @@ SHELL := /bin/bash
 HELM  := helm
 TASK  := build
 
-EXCLUDES := helm-toolkit doc tests tools logs tmp zuul.d releasenotes
+EXCLUDES := helm-toolkit doc tests tools logs tmp zuul.d releasenotes roles
 CHARTS := helm-toolkit $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
 
 .PHONY: $(EXCLUDES) $(CHARTS)

tools/deployment/component/compute-kit/compute-kit.sh

@@ -95,7 +95,7 @@ helm upgrade --install neutron ${OSH_HELM_REPO}/neutron \
     ${OSH_EXTRA_HELM_ARGS_NEUTRON}
 
 #NOTE: Wait for deploy
-helm osh wait-for-pods openstack
+helm osh wait-for-pods openstack 1200
 
 #NOTE: Validate Deployment info
 export OS_CLOUD=openstack_helm

tools/gate/playbooks/deploy-env-kubespray.yaml

@@ -0,0 +1,144 @@
+- hosts: all
+  become: true
+  gather_facts: true
+  tasks:
+    - name: Install prerequisites
+      include_role:
+        name: deploy-env
+        tasks_from: prerequisites
+
+    - name: Create loopback devices
+      include_role:
+        name: deploy-env
+        tasks_from: loopback_devices
+      when:
+        - loopback_setup
+        - inventory_hostname in (groups['k8s_cluster'] | default([]))
+
+- hosts: primary
+  become: false
+  gather_facts: true
+  vars:
+    home_dir: /home/zuul
+    ansible_user: zuul
+  tasks:
+    - name: Clone Kubespray repo
+      shell: |
+        set -x
+        git clone https://github.com/kubernetes-sigs/kubespray.git
+        cd kubespray
+        git checkout -b release-2.25
+        git reset --hard v2.25.0
+      args:
+        chdir: "{{ home_dir }}"
+
+    - name: Install Kubespray Python dependencies
+      become: true
+      pip:
+        chdir: "{{ home_dir }}/kubespray"
+        requirements: requirements.txt
+
+    - name: Prepare Kubespray inventory (not Zuul job inventory)
+      shell: |
+        #!/bin/bash
+        set -x
+        python3 contrib/inventory_builder/inventory.py {{ groups['k8s_cluster'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | join(' ') }}
+      args:
+        chdir: "{{ home_dir }}/kubespray"
+      environment:
+        KUBE_MASTERS: "1"
+
+    - name: Prepare Kubespray variables
+      shell: |
+        #!/bin/bash
+        set -x
+        tee inventory/sample/group_vars/all/xxx.yaml <<EOF
+        ansible_user: {{ ansible_user }}
+        ansible_ssh_private_key_file: {{ home_dir }}/.ssh/id_rsa
+        ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+        upstream_dns_servers:
+          - 8.8.8.8
+        override_system_hostname: false
+        EOF
+
+        tee inventory/sample/group_vars/k8s_cluster/xxx.yaml << EOF
+        kube_version: "{{ kube_version_kubespray }}"
+        kube_network_plugin: flannel
+        kube_service_addresses: "{{ kubeadm.service_cidr }}"
+        kube_pods_subnet: "{{ kubeadm.pod_network_cidr }}"
+
+        kubeconfig_localhost: true
+        kubeconfig_localhost_ansible_host: true
+
+        enable_nodelocaldns: false
+        resolvconf_mode: none
+
+        kube_override_hostname: >-
+          {% raw %}{{ ansible_hostname }}{% endraw %}
+
+        EOF
+      args:
+        chdir: "{{ home_dir }}/kubespray"
+
+    - name: Deploy Kubernetes
+      shell: |
+        #!/bin/bash
+        set -x
+        ansible-playbook -i inventory/sample/hosts.yaml --become --become-user=root cluster.yml
+      args:
+        chdir: /home/zuul/kubespray
+
+    - name: Copy kubectl config to localhost (will be used in the following tasks)
+      synchronize:
+        mode: pull
+        src: /home/zuul/kubespray/inventory/sample/artifacts/admin.conf
+        dest: /tmp/kube_config
+
+- hosts: primary
+  become: true
+  gather_facts: true
+  tasks:
+    - name: Install Docker
+      include_role:
+        name: deploy-env
+        tasks_from: containerd
+
+    - name: Install and configure Kubectl and Helm
+      include_role:
+        name: deploy-env
+        tasks_from: k8s_client
+
+    - name: Deploy Metallb on K8s
+      include_role:
+        name: deploy-env
+        tasks_from: metallb
+
+    - name: Create Openstack Metallb endpoint
+      include_role:
+        name: deploy-env
+        tasks_from: openstack_metallb_endpoint
+
+- hosts: all
+  become: true
+  gather_facts: true
+  tasks:
+    - name: Create client-to-cluster wireguard tunnel
+      include_role:
+        name: deploy-env
+        tasks_from: client_cluster_tunnel
+
+    - name: Install Docker
+      include_role:
+        name: deploy-env
+        tasks_from: containerd
+      when:
+        - openstack_provider_gateway_setup
+        - inventory_hostname in (groups['k8s_control_plane'] | default([]))
+
+    - name: Deploy Openstack provider gateway
+      include_role:
+        name: deploy-env
+        tasks_from: openstack_provider_gateway
+      when:
+        - openstack_provider_gateway_setup
+        - inventory_hostname in (groups['k8s_control_plane'] | default([]))

tools/gate/playbooks/deploy-env.yaml

@@ -2,7 +2,4 @@
   become: true
   gather_facts: true
   roles:
-    - ensure-python
-    - ensure-pip
-    - clear-firewall
     - deploy-env

tools/gate/playbooks/prepare-hosts.yaml

@@ -1,3 +1,6 @@
 - hosts: all
   roles:
     - start-zuul-console
+    - ensure-python
+    - ensure-pip
+    - clear-firewall

zuul.d/2024.1.yaml

@@ -77,6 +77,18 @@
         container_distro_version: jammy
         feature_gates: metallb
 
+- job:
+    name: openstack-helm-compute-kit-kubespray-2024-1-ubuntu_jammy
+    parent: openstack-helm-compute-kit-kubespray
+    nodeset: openstack-helm-1node-3nodes-ubuntu_jammy
+    vars:
+      metallb_setup: true
+      osh_params:
+        openstack_release: "2024.1"
+        container_distro_name: ubuntu
+        container_distro_version: jammy
+        feature_gates: metallb
+
 - job:
     name: openstack-helm-compute-kit-cilium-metallb-2024-1-ubuntu_jammy
     parent: openstack-helm-compute-kit

zuul.d/base.yaml

@@ -69,6 +69,17 @@
       run_helm_tests: "no"
       openstack_provider_gateway_setup: true
 
+- job:
+    name: openstack-helm-deploy-kubespray
+    parent: openstack-helm-deploy
+    abstract: true
+    run:
+      - tools/gate/playbooks/deploy-env-kubespray.yaml
+      - tools/gate/playbooks/run-scripts.yaml
+    vars:
+      metallb_setup: true
+      kube_version_kubespray: "v1.29.5"
+
 - job:
     name: openstack-helm-compute-kit
     parent: openstack-helm-deploy
@@ -79,7 +90,33 @@
         - ./tools/deployment/common/prepare-charts.sh
         - ./tools/deployment/common/setup-client.sh
         - ./tools/deployment/common/ingress.sh
-        - ./tools/deployment/component/common/rabbitmq.sh
+        - export VOLUME_HELM_ARGS="--set volume.enabled=false"; ./tools/deployment/component/common/rabbitmq.sh
+        - ./tools/deployment/component/common/mariadb.sh
+        - ./tools/deployment/component/common/memcached.sh
+        - ./tools/deployment/component/keystone/keystone.sh
+        - ./tools/deployment/component/heat/heat.sh
+        - export GLANCE_BACKEND=local; ./tools/deployment/component/glance/glance.sh
+        - ./tools/deployment/component/compute-kit/openvswitch.sh
+        - ./tools/deployment/component/compute-kit/libvirt.sh
+        - ./tools/deployment/component/compute-kit/compute-kit.sh
+        - export OSH_TEST_TIMEOUT=1200;./tools/deployment/common/run-helm-tests.sh neutron
+        - ./tools/deployment/common/run-helm-tests.sh nova
+        - ./tools/deployment/common/run-helm-tests.sh glance
+        - ./tools/deployment/common/run-helm-tests.sh keystone
+        - ./tools/deployment/common/use-it.sh
+        - ./tools/deployment/common/force-cronjob-run.sh
+
+- job:
+    name: openstack-helm-compute-kit-kubespray
+    parent: openstack-helm-deploy-kubespray
+    abstract: true
+    vars:
+      gate_scripts:
+        - ./tools/deployment/common/prepare-k8s.sh
+        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/setup-client.sh
+        - ./tools/deployment/common/ingress.sh
+        - export VOLUME_HELM_ARGS="--set volume.enabled=false"; ./tools/deployment/component/common/rabbitmq.sh
         - ./tools/deployment/component/common/mariadb.sh
         - ./tools/deployment/component/common/memcached.sh
         - ./tools/deployment/component/keystone/keystone.sh

zuul.d/nodesets.yaml

@@ -146,6 +146,34 @@
         nodes:
           - node-2
 
+- nodeset:
+    name: openstack-helm-1node-3nodes-ubuntu_jammy
+    nodes:
+      - name: primary
+        label: ubuntu-jammy
+      - name: node-1
+        label: ubuntu-jammy
+      - name: node-2
+        label: ubuntu-jammy
+      - name: node-3
+        label: ubuntu-jammy
+    groups:
+      - name: primary
+        nodes:
+          - primary
+      - name: k8s_cluster
+        nodes:
+          - node-1
+          - node-2
+          - node-3
+      - name: k8s_control_plane
+        nodes:
+          - node-1
+      - name: k8s_nodes
+        nodes:
+          - node-2
+          - node-3
+
 - nodeset:
     name: openstack-helm-1node-32GB-ubuntu_jammy
     nodes:

zuul.d/project.yaml

@@ -43,6 +43,7 @@
         - openstack-helm-compute-kit-flannel-metallb-2024-1-ubuntu_jammy  # 1 node + 2 nodes
         - openstack-helm-horizon-2024-1-ubuntu_jammy  # 1 node
         - openstack-helm-tacker-2024-1-ubuntu_jammy
+        - openstack-helm-compute-kit-kubespray-2024-1-ubuntu_jammy
     gate:
       jobs:
         - openstack-helm-lint