[deploy-env] Switch to kubeadm role

Change-Id: I269243ff6a8e4a9c8761aa518c149602d3cba8ef

tools/deployment/003-prepare-k8s.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Add labels to the core namespaces & nodes
+kubectl label --overwrite namespace default name=default
+kubectl label --overwrite namespace kube-system name=kube-system
+kubectl label --overwrite namespace kube-public name=kube-public
+kubectl label --overwrite nodes --all openstack-control-plane=enabled
+kubectl label --overwrite nodes --all openstack-compute-node=enabled
+kubectl label --overwrite nodes --all openvswitch=enabled
+kubectl label --overwrite nodes --all linuxbridge=enabled
+kubectl label --overwrite nodes --all ceph-mon=enabled
+kubectl label --overwrite nodes --all ceph-osd=enabled
+kubectl label --overwrite nodes --all ceph-mds=enabled
+kubectl label --overwrite nodes --all ceph-rgw=enabled
+kubectl label --overwrite nodes --all ceph-mgr=enabled
+# We deploy l3 agent only on the node where we run test scripts.
+# In this case virtual router will be created only on this node
+# and we don't need L2 overlay (will be implemented later).
+kubectl label --overwrite nodes -l "node-role.kubernetes.io/control-plane" l3-agent=enabled
+
+for NAMESPACE in ceph mariadb-operator utility ucp openstack osh-infra; do
+tee /tmp/${NAMESPACE}-ns.yaml << EOF
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    kubernetes.io/metadata.name: ${NAMESPACE}
+    name: ${NAMESPACE}
+  name: ${NAMESPACE}
+EOF
+
+kubectl apply -f /tmp/${NAMESPACE}-ns.yaml
+done

tools/deployment/010-ceph-utility.sh

@@ -3,15 +3,9 @@ CURRENT_DIR="$(pwd)"
  : "${OSH_PATH:="../openstack-helm"}"
  : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
 
-# remove previous loop devices if any
 
-if [ -f /etc/systemd/system/local-fs.target.wants/loops-setup.service ]
-then
-  sudo systemctl stop loops-setup
-fi
-
-cd "${OSH_PATH}" || exit
-sudo bash -c "./tools/deployment/component/ceph/ceph.sh"
+cd "${OSH_INFRA_PATH}" || exit
+.//tools/deployment/ceph/ceph.sh
 
 namespace="utility"
 : ${OSH_EXTRA_HELM_ARGS:=""}

tools/deployment/040-mysqlclient-utility.sh

@@ -29,10 +29,6 @@ manifests:
   cron_job_mariadb_backup: true
   secret_backup_restore: true
   pvc_backup: true
-volume:
-  class_name: standard
-  backup:
-    class_name: standard
 EOF
 
 export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../openstack-helm-infra"}"}"

tools/deployment/060-postgresql-utility.sh

@@ -32,8 +32,6 @@ helm upgrade --install postgresql ./postgresql \
     --set storage.pvc.size=1Gi \
     --set storage.pvc.enabled=true \
     --set pod.replicas.server=1 \
-    --set storage.pvc.class_name=standard \
-    --set storage.archive_pvc.class_name=standard \
     ${OSH_INFRA_EXTRA_HELM_ARGS} \
     ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL}
 

tools/deployment/sleep.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+env
+
+while true; do sleep 10; done

tools/gate/playbooks/airship-run-scripts.yaml

@@ -0,0 +1,75 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: primary
+  tasks:
+    - name: Override images
+      when: buildset_registry is defined
+      vars:
+        work_dir: "{{ zuul.project.src_dir }}"
+      block:
+        - name: Buildset registry alias
+          include_role:
+            name: deploy-env
+            tasks_from: buildset_registry_alias
+
+        - name: Print zuul
+          debug:
+            var: zuul
+
+        - name: Override proposed images from artifacts
+          shell: >
+            find {{ override_paths | join(" ") }} -type f -exec sed -Ei
+            "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
+          loop: "{{ zuul.artifacts | default([]) }}"
+          args:
+            chdir: "{{ work_dir }}"
+          loop_control:
+            loop_var: zj_zuul_artifact
+          when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
+          vars:
+            tag: "{{ zj_zuul_artifact.metadata.tag }}"
+            repo: "{{ zj_zuul_artifact.metadata.repository }}"
+            override_paths:
+              - ../openstack-helm*/*/values*
+              - ../openstack-helm-infra/tools/deployment/
+
+        - name: Diff
+          shell: |
+              set -ex;
+              for dir in openstack-helm openstack-helm-infra; do
+                path="{{ work_dir }}/../${dir}/"
+                if [ ! -d "${path}" ]; then continue; fi
+                echo "${dir} diff"
+                cd "${path}"; git diff; cd -;
+              done
+
+    - name: "creating directory for run artifacts"
+      file:
+        path: "/tmp/artifacts"
+        state: directory
+
+    - name: Run gate scripts
+      include_role:
+        name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
+      vars:
+        workload: "{{ [item] | flatten }}"
+      loop: "{{ gate_scripts }}"
+
+    - name: "Downloads artifacts to executor"
+      synchronize:
+        src: "/tmp/artifacts"
+        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+        mode: pull
+      ignore_errors: True
+...

tools/gate/playbooks/deploy-env.yaml

@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  become: true
+  gather_facts: true
+
+  roles:
+    - ensure-python
+    - ensure-pip
+    - ensure-tox
+    - clear-firewall
+    - deploy-apparmor
+    - deploy-selenium
+    - deploy-env
+
+  tasks:
+    - name: Install Packaging python module for tools/airship
+      block:
+        - pip:
+            name: packaging
+            version: 23.1
+            executable: pip3
+      become: True
+
+...

tools/gate/playbooks/mount-volumes.yaml

@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - mount-extra-volume
+...

tools/gate/playbooks/osh-infra-collect-logs.yaml

@@ -0,0 +1,43 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - gather-host-logs
+  tags:
+    - gather-host-logs
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+  tags:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+...

tools/gate/playbooks/prepare-hosts.yaml

@@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - start-zuul-console
+...

tools/gate/playbooks/roles/airship-run-script-set/defaults/main.yaml

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+...

tools/gate/playbooks/roles/airship-run-script-set/tasks/main.yaml

@@ -0,0 +1,64 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: "Run script set {{ workload }}"
+      shell: |
+        set -xe;
+        {{ gate_script_path }}
+      loop: "{{ workload }}"
+      loop_control:
+        loop_var: gate_script_path
+        pause: 5
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+      environment:
+        CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+        POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+        OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+        CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+        CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+        FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+        RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+        PL_SITE: "{{ site | default('airskiff') }}"
+        HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+        HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+        COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+      # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with
+      # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637
+      async: 3600
+      poll: 0
+      register: async_results
+
+    - name: Wait for script set to finish
+      async_status:
+        jid: '{{ item.ansible_job_id }}'
+      register: jobs
+      until: jobs.finished
+      delay: 5
+      retries: 360
+      loop: "{{ async_results.results }}"
+
+  always:
+    - name: Print script set output
+      shell: |
+          # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status
+          echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}'
+      loop: "{{ jobs.results }}"
+...

tools/gate/playbooks/roles/airship-run-script/defaults/main.yaml

@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+...

tools/gate/playbooks/roles/airship-run-script/tasks/main.yaml

@@ -0,0 +1,40 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "Run script {{ workload[0] }}"
+  shell: |
+    set -xe;
+    {{ gate_script_path }}
+  vars:
+    gate_script_path: "{{ workload[0] }}"
+  args:
+    chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+  environment:
+    CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+    POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+    zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+    OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+    OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+    OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+    OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+    CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+    CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+    FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+    RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+    PL_SITE: "{{ site | default('airskiff') }}"
+    HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+    HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+    COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+...

tools/helm_tk.sh

@@ -17,7 +17,7 @@
 set -eux
 
 HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"13c1d8cd3866bb026cce6e5d80555edfd0afa845"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"cfff60ec10a6c386f38db79bb9f59a552c2b032f"}
 
 TMP_DIR=$(mktemp -d)
 

zuul.d/base.yaml

@@ -19,8 +19,8 @@
         - airship-porthole-linter
         - openstack-tox-pep8
         - openstack-tox-docs
-        - airship-porthole-openstack-tox-py38
-        - airship-porthole-openstack-tox-cover
+        # - airship-porthole-openstack-tox-py38
+        # - airship-porthole-openstack-tox-cover
         - airship-porthole-images-focal-build-gate-calicoctl-utility
         - airship-porthole-images-focal-build-gate-ceph-utility
         - airship-porthole-images-focal-build-gate-compute-utility
@@ -28,23 +28,20 @@
         - airship-porthole-images-focal-build-gate-mysqlclient-utility
         - airship-porthole-images-focal-build-gate-openstack-utility
         - airship-porthole-images-focal-build-gate-postgresql-utility
-        - airship-porthole-deploy-functional-tests-ubuntu_focal
-#        - airship-porthole-images-bionic-build-gate-calicoctl-utility
-#        - airship-porthole-images-bionic-build-gate-ceph-utility
-#        - airship-porthole-images-bionic-build-gate-compute-utility
-#        - airship-porthole-images-bionic-build-gate-etcdctl-utility
-#        - airship-porthole-images-bionic-build-gate-mysqlclient-utility
-#        - airship-porthole-images-bionic-build-gate-openstack-utility
-#        - airship-porthole-images-bionic-build-gate-postgresql-utility
-#        - airship-porthole-deploy-functional-tests-ubuntu_bionic
+        # - airship-porthole-deploy-functional-tests-ubuntu_focal
+        - airship-porthole-openstack-tox-py38-kubeadm
+        - airship-porthole-openstack-tox-cover-kubeadm
+        - airship-porthole-deploy-functional-tests-ubuntu_focal-kubeadm
+
 
     gate:
       jobs:
         - airship-porthole-linter
         - openstack-tox-pep8
         - openstack-tox-docs
-        - airship-porthole-openstack-tox-py38
-        - airship-porthole-openstack-tox-cover
+        - airship-porthole-openstack-tox-py38-kubeadm
+        - airship-porthole-openstack-tox-cover-kubeadm
+        - airship-porthole-deploy-functional-tests-ubuntu_focal-kubeadm
         - airship-porthole-images-focal-build-gate-calicoctl-utility
         - airship-porthole-images-focal-build-gate-ceph-utility
         - airship-porthole-images-focal-build-gate-compute-utility
@@ -52,15 +49,7 @@
         - airship-porthole-images-focal-build-gate-mysqlclient-utility
         - airship-porthole-images-focal-build-gate-openstack-utility
         - airship-porthole-images-focal-build-gate-postgresql-utility
-        - airship-porthole-deploy-functional-tests-ubuntu_focal
-#        - airship-porthole-images-bionic-build-gate-calicoctl-utility
-#        - airship-porthole-images-bionic-build-gate-ceph-utility
-#        - airship-porthole-images-bionic-build-gate-compute-utility
-#        - airship-porthole-images-bionic-build-gate-etcdctl-utility
-#        - airship-porthole-images-bionic-build-gate-mysqlclient-utility
-#        - airship-porthole-images-bionic-build-gate-openstack-utility
-#        - airship-porthole-images-bionic-build-gate-postgresql-utility
-#        - airship-porthole-deploy-functional-tests-ubuntu_bionic
+
 
 
     post:
@@ -72,13 +61,6 @@
         - airship-porthole-images-publish-mysqlclient-utility
         - airship-porthole-images-publish-openstack-utility
         - airship-porthole-images-publish-postgresql-utility
-#        - airship-porthole-images-bionic-publish-calicoctl-utility
-#        - airship-porthole-images-bionic-publish-ceph-utility
-#        - airship-porthole-images-bionic-publish-compute-utility
-#        - airship-porthole-images-bionic-publish-etcdctl-utility
-#        - airship-porthole-images-bionic-publish-mysqlclient-utility
-#        - airship-porthole-images-bionic-publish-openstack-utility
-#        - airship-porthole-images-bionic-publish-postgresql-utility
 
 
 - nodeset:
@@ -93,36 +75,69 @@
       - name: primary
         label: ubuntu-focal
 
-- job:
-    name: airship-porthole-openstack-tox-py38
-    parent: openstack-tox-py38
-    description: Runs cover job on focal
-    nodeset: airship-porthole-focal-single-node
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    timeout: 7200
-    vars:
-      distro_suffix: ubuntu_focal
-      gate_scripts:
-        - ./tools/deployment/005-calicoctl-utility.sh
-        - ./tools/deployment/010-ceph-utility.sh
-        - ./tools/deployment/020-compute-utility.sh
-        - ./tools/deployment/030-etcdctl-utility.sh
-        - ./tools/deployment/040-mysqlclient-utility.sh
-        - ./tools/deployment/050-openstack-utility.sh
-        - ./tools/deployment/060-postgresql-utility.sh
-      args:
-        chdir: ../porthole
 
 - job:
-    name: airship-porthole-openstack-tox-cover
-    parent: openstack-tox-cover
-    description: Runs cover job on focal
-    nodeset: airship-porthole-focal-single-node
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    timeout: 7200
+    name: airship-porthole-openstack-tox-py38-kubeadm
+    parent: openstack-tox-py38
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    roles:
+      - zuul: openstack/openstack-helm-infra
+      - zuul: zuul/zuul-jobs
+    required-projects:
+      - name: openstack/openstack-helm
+      - name: openstack/openstack-helm-infra
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^site/seaworthy/.*$
+      - ^site/airsloop/.*$
+      - ^site/aiab/.*$
+    timeout: 10800
+    pre-run:
+      - tools/gate/playbooks/prepare-hosts.yaml
+      - tools/gate/playbooks/mount-volumes.yaml
+      - tools/gate/playbooks/deploy-env.yaml
+      - tools/gate/playbooks/airship-run-scripts.yaml
+    post-run:
+      - tools/gate/playbooks/osh-infra-collect-logs.yaml
     vars:
+      extra_volume:
+        size: 80G
+        type: Linux
+        mount_point: /opt/ext_vol
+      docker:
+        root_path: "/opt/ext_vol/docker"
+      containerd:
+        root_path: "/opt/ext_vol/containerd"
+      kubeadm:
+        pod_network_cidr: "10.244.0.0/24"
+        service_cidr: "10.96.0.0/16"
+      loopback_setup: true
+      loopback_device: /dev/loop100
+      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
+      ceph_osd_data_device: /dev/loop100
+      kube_version: "1.28.4-1.1"
+      calico_version: "v3.27.0"
+      coredns_version: "v1.11.1"
+      helm_version: "v3.13.2"
+      yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
+      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
+      zuul_osh_relative_path: ../../openstack/openstack-helm
+      gate_scripts_relative_path: .
+      run_helm_tests: "no"
       distro_suffix: ubuntu_focal
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
       gate_scripts:
+        - ./tools/deployment/002-build-charts.sh
+        - ./tools/deployment/003-prepare-k8s.sh
+        - ./tools/deployment/004-make-images.sh
         - ./tools/deployment/005-calicoctl-utility.sh
         - ./tools/deployment/010-ceph-utility.sh
         - ./tools/deployment/020-compute-utility.sh
@@ -130,8 +145,119 @@
         - ./tools/deployment/040-mysqlclient-utility.sh
         - ./tools/deployment/050-openstack-utility.sh
         - ./tools/deployment/060-postgresql-utility.sh
-      args:
-        chdir: ../porthole
+
+
+# - job:
+#     name: airship-porthole-openstack-tox-py38
+#     parent: openstack-tox-py38
+#     description: Runs cover job on focal
+#     nodeset: airship-porthole-focal-single-node
+#     pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
+#     timeout: 7200
+#     vars:
+#       distro_suffix: ubuntu_focal
+#       gate_scripts:
+#         - ./tools/deployment/005-calicoctl-utility.sh
+#         - ./tools/deployment/010-ceph-utility.sh
+#         - ./tools/deployment/020-compute-utility.sh
+#         - ./tools/deployment/030-etcdctl-utility.sh
+#         - ./tools/deployment/040-mysqlclient-utility.sh
+#         - ./tools/deployment/050-openstack-utility.sh
+#         - ./tools/deployment/060-postgresql-utility.sh
+#       args:
+#         chdir: ../porthole
+
+- job:
+    name: airship-porthole-openstack-tox-cover-kubeadm
+    parent: openstack-tox-cover
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    roles:
+      - zuul: openstack/openstack-helm-infra
+      - zuul: zuul/zuul-jobs
+    required-projects:
+      - name: openstack/openstack-helm
+      - name: openstack/openstack-helm-infra
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^site/seaworthy/.*$
+      - ^site/airsloop/.*$
+      - ^site/aiab/.*$
+    timeout: 10800
+    pre-run:
+      - tools/gate/playbooks/prepare-hosts.yaml
+      - tools/gate/playbooks/mount-volumes.yaml
+      - tools/gate/playbooks/deploy-env.yaml
+      - tools/gate/playbooks/airship-run-scripts.yaml
+    post-run:
+      - tools/gate/playbooks/osh-infra-collect-logs.yaml
+    vars:
+      extra_volume:
+        size: 80G
+        type: Linux
+        mount_point: /opt/ext_vol
+      docker:
+        root_path: "/opt/ext_vol/docker"
+      containerd:
+        root_path: "/opt/ext_vol/containerd"
+      kubeadm:
+        pod_network_cidr: "10.244.0.0/24"
+        service_cidr: "10.96.0.0/16"
+      loopback_setup: true
+      loopback_device: /dev/loop100
+      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
+      ceph_osd_data_device: /dev/loop100
+      kube_version: "1.28.4-1.1"
+      calico_version: "v3.27.0"
+      coredns_version: "v1.11.1"
+      helm_version: "v3.13.2"
+      yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
+      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
+      zuul_osh_relative_path: ../../openstack/openstack-helm
+      gate_scripts_relative_path: .
+      run_helm_tests: "no"
+      distro_suffix: ubuntu_focal
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      gate_scripts:
+        - ./tools/deployment/002-build-charts.sh
+        - ./tools/deployment/003-prepare-k8s.sh
+        - ./tools/deployment/004-make-images.sh
+        - ./tools/deployment/005-calicoctl-utility.sh
+        - ./tools/deployment/010-ceph-utility.sh
+        - ./tools/deployment/020-compute-utility.sh
+        - ./tools/deployment/030-etcdctl-utility.sh
+        - ./tools/deployment/040-mysqlclient-utility.sh
+        - ./tools/deployment/050-openstack-utility.sh
+        - ./tools/deployment/060-postgresql-utility.sh
+
+
+
+# - job:
+#     name: airship-porthole-openstack-tox-cover
+#     parent: openstack-tox-cover
+#     description: Runs cover job on focal
+#     nodeset: airship-porthole-focal-single-node
+#     pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
+#     timeout: 7200
+#     vars:
+#       distro_suffix: ubuntu_focal
+#       gate_scripts:
+#         - ./tools/deployment/005-calicoctl-utility.sh
+#         - ./tools/deployment/010-ceph-utility.sh
+#         - ./tools/deployment/020-compute-utility.sh
+#         - ./tools/deployment/030-etcdctl-utility.sh
+#         - ./tools/deployment/040-mysqlclient-utility.sh
+#         - ./tools/deployment/050-openstack-utility.sh
+#         - ./tools/deployment/060-postgresql-utility.sh
+#       args:
+#         chdir: ../porthole
 
 - job:
     name: airship-porthole-images
@@ -139,7 +265,7 @@
     timeout: 7200
     post-timeout: 7200
     run: tools/gate/playbooks/docker-image-build.yaml
-    nodeset: airship-porthole-focal-single-node
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
     vars:
       publish: false
       tags:
@@ -157,24 +283,74 @@
       Lints files for trailing whitespace
     run: tools/gate/playbooks/zuul-linter.yaml
     timeout: 300
-    nodeset: airship-porthole-focal-single-node
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
     vars:
       args:
         chdir: ../porthole
 
 - job:
-    name: airship-porthole-deploy-functional-tests
+    name:  airship-porthole-deploy-functional-tests-kubeadm
     abstract: true
     description: Deploys all UC's and Executes functional tests
-    nodeset: airship-porthole-focal-single-node
-    timeout: 7200
-    run:
-      - tools/gate/playbooks/airship-porthole-gate-runner.yaml
-      - tools/gate/playbooks/make-functional-tests.yaml
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    roles:
+      - zuul: openstack/openstack-helm-infra
+      - zuul: zuul/zuul-jobs
+    required-projects:
+      - name: openstack/openstack-helm
+      - name: openstack/openstack-helm-infra
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^site/seaworthy/.*$
+      - ^site/airsloop/.*$
+      - ^site/aiab/.*$
+    timeout: 10800
+    pre-run:
+      - tools/gate/playbooks/prepare-hosts.yaml
+      - tools/gate/playbooks/mount-volumes.yaml
+      - tools/gate/playbooks/deploy-env.yaml
+      - tools/gate/playbooks/airship-run-scripts.yaml
     post-run:
-      - tools/gate/playbooks/airship-porthole-collect-logs.yaml
+      - tools/gate/playbooks/osh-infra-collect-logs.yaml
     vars:
+      extra_volume:
+        size: 80G
+        type: Linux
+        mount_point: /opt/ext_vol
+      docker:
+        root_path: "/opt/ext_vol/docker"
+      containerd:
+        root_path: "/opt/ext_vol/containerd"
+      kubeadm:
+        pod_network_cidr: "10.244.0.0/24"
+        service_cidr: "10.96.0.0/16"
+      loopback_setup: true
+      loopback_device: /dev/loop100
+      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
+      ceph_osd_data_device: /dev/loop100
+      kube_version: "1.28.4-1.1"
+      calico_version: "v3.27.0"
+      coredns_version: "v1.11.1"
+      helm_version: "v3.13.2"
+      yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
+      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
+      zuul_osh_relative_path: ../../openstack/openstack-helm
+      gate_scripts_relative_path: .
+      run_helm_tests: "no"
+      distro_suffix: ubuntu_focal
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
       gate_scripts:
+        - ./tools/deployment/002-build-charts.sh
+        - ./tools/deployment/003-prepare-k8s.sh
+        - ./tools/deployment/004-make-images.sh
         - ./tools/deployment/005-calicoctl-utility.sh
         - ./tools/deployment/010-ceph-utility.sh
         - ./tools/deployment/020-compute-utility.sh
@@ -182,28 +358,53 @@
         - ./tools/deployment/040-mysqlclient-utility.sh
         - ./tools/deployment/050-openstack-utility.sh
         - ./tools/deployment/060-postgresql-utility.sh
-      args:
-        chdir: ../porthole
 
 - job:
-    name: airship-porthole-deploy-functional-tests-ubuntu_focal
-    parent: airship-porthole-deploy-functional-tests
+    name: airship-porthole-deploy-functional-tests-ubuntu_focal-kubeadm
+    parent: airship-porthole-deploy-functional-tests-kubeadm
+    run: tools/gate/playbooks/make-functional-tests.yaml
     vars:
+      args:
+        chdir: ../porthole
       distro_suffix: ubuntu_focal
       osh_params:
         openstack_release: yoga
         container_distro_name: ubuntu
         container_distro_version: focal
 
-- job:
-    name: airship-porthole-deploy-functional-tests-ubuntu_bionic
-    parent: airship-porthole-deploy-functional-tests
-    vars:
-      distro_suffix: ubuntu_bionic
-      osh_params:
-        openstack_release: yoga
-        container_distro_name: ubuntu
-        container_distro_version: bionic
+
+# - job:
+#     name: airship-porthole-deploy-functional-tests
+#     abstract: true
+#     description: Deploys all UC's and Executes functional tests
+#     nodeset: airship-porthole-focal-single-node
+#     timeout: 7200
+#     run:
+#       - tools/gate/playbooks/airship-porthole-gate-runner.yaml
+#       - tools/gate/playbooks/make-functional-tests.yaml
+#     post-run:
+#       - tools/gate/playbooks/airship-porthole-collect-logs.yaml
+#     vars:
+#       gate_scripts:
+#         - ./tools/deployment/005-calicoctl-utility.sh
+#         - ./tools/deployment/010-ceph-utility.sh
+#         - ./tools/deployment/020-compute-utility.sh
+#         - ./tools/deployment/030-etcdctl-utility.sh
+#         - ./tools/deployment/040-mysqlclient-utility.sh
+#         - ./tools/deployment/050-openstack-utility.sh
+#         - ./tools/deployment/060-postgresql-utility.sh
+#       args:
+#         chdir: ../porthole
+
+# - job:
+#     name: airship-porthole-deploy-functional-tests-ubuntu_focal
+#     parent: airship-porthole-deploy-functional-tests
+#     vars:
+#       distro_suffix: ubuntu_focal
+#       osh_params:
+#         openstack_release: yoga
+#         container_distro_name: ubuntu
+#         container_distro_version: focal
 
 
 - secret: