Porthole upgrades

This PS delivers the following changes:

- adds values_overrides folders into utility binaries charts for
  flexible testing the charts and images;
- values_overrides added to .helmignore files for every chart;
- added munch python package into bionic images for compartibility with
  newer openstack client dependences;
- improved Makefile clean section;
- uses treasuremap v1.9 deploy-k8s.sh script to deploy test k8s cluster;
- adds appropriate changes to deploy k8s script, setup apparmor;
- porthole utility charts deployment script have been enriched with and
  ability to control additional values from values_overrides folders of
  each chart;
- added env-variables.sh and get-values-overrides.sh to power up the
  changes describes in previous point;
- removed deploy.sh script for clarity of actions in playbooks;
- improved airship-porthole-gate-runner playbook;
- removed install-deps playbook that was used to pre-setup zuul gate;
- united make-unit-tests and make-feature-tests playbooks into one
  make-functional tests playbook;
- helm version has been bumped up to 3.11.1;
- improved helm_tk.sh script in order to solve issues with multiple
  charts build process;
- bumped up version of stable helm-toolkit used for charts build;
- airship-porthole-gate-runner playbook is now used as pre-run script
  for tox profiles gates and as main run playbook for functional tests
  gate;

Change-Id: I202c573ff85ed2b80191057b537a65a5f5d4600e

.gitignore

@@ -69,9 +69,9 @@ releasenotes/build
 **/*.tgz
 **/_partials.tpl
 **/_globals.tpl
-/charts/deps/*/
 /charts/deps/*
 
+
 # Gate and Check Logs
 logs/
 tmp/

Makefile

@@ -22,7 +22,7 @@ IMAGE_NAME        ?= $@
 IMAGE_TAG         ?= latest
 BUILD_DIR         := $(shell mktemp -d)
 HELM              ?= $(BUILD_DIR)/helm
-CHARTS            := $(patsubst charts/%/.,%,$(wildcard charts/*/.))
+CHARTS            := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
 IMAGES            := $(addprefix porthole-,$(patsubst images/%/.,%,$(wildcard images/*/.)))
 PROXY             ?= http://proxy.foo.com:8000
 NO_PROXY          ?= localhost,127.0.0.1,.svc.cluster.local
@@ -125,6 +125,9 @@ clean:
 	rm -f charts/*.tgz
 	rm -f charts/*/requirements.lock
 	rm -rf charts/*/charts
+	rm -rf charts/deps/*
+	rm -rf .tox
+	rm -rf cover
 
 run_images:
 	@echo "Not implemented." >&2; exit 2

charts/calicoctl-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/calicoctl-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    calicoctl_utility: quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_bionic
+...

charts/calicoctl-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    calicoctl_utility: quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_focal
+...

charts/ceph-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/ceph-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    ceph_utility: quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_bionic
+...

charts/ceph-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    ceph_utility: quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_focal
+...

charts/compute-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/compute-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    compute_utility: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_bionic
+...

charts/compute-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    compute_utility: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_focal
+...

charts/etcdctl-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/etcdctl-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    etcdctl_utility: quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_bionic
+...

charts/etcdctl-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    etcdctl_utility: quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_focal
+...

charts/mysqlclient-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/mysqlclient-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    mysqlclient_utility: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
+...

charts/mysqlclient-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    mysqlclient_utility: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal
+...

charts/openstack-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/openstack-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    openstack_utility: quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_bionic
+...

charts/openstack-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    openstack_utility: quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_focal
+...

charts/postgresql-utility/.helmignore

@@ -0,0 +1 @@
+values_overrides

charts/postgresql-utility/values_overrides/images-ubuntu_bionic.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    postgresql_utility: quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic
+...

charts/postgresql-utility/values_overrides/images-ubuntu_focal.yaml

@@ -0,0 +1,5 @@
+---
+images:
+  tags:
+    postgresql_utility: quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_focal
+...

images/etcdctl-utility/Dockerfile.ubuntu_bionic

@@ -36,6 +36,7 @@ RUN set -xe \
        rclone \
     && pip3 install --upgrade pip \
     && pip3 install \
+        munch \
         oslo.rootwrap==6.2.0 \
         python-openstackclient==5.4.0 \
         python-swiftclient==3.10.1 \

images/mysqlclient-utility/Dockerfile.ubuntu_bionic

@@ -36,6 +36,7 @@ RUN set -xe \
        rclone \
     && pip3 install --upgrade pip \
     && pip3 install \
+         munch \
          oslo.rootwrap==6.2.0 \
          python-openstackclient==5.4.0 \
          python-swiftclient==3.10.1 \
@@ -43,7 +44,7 @@ RUN set -xe \
     && apt-get clean -y \
     && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \
        | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl"  \
-    && curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \
+    && curl --silent -L https://downloads.percona.com/downloads/percona-toolkit/3.5.1/binary/tarball/percona-toolkit-3.5.1_x86_64.tar.gz \
        | tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \
     && rm -rf \
       /tmp/* \

images/postgresql-utility/Dockerfile.ubuntu_bionic

@@ -34,6 +34,7 @@ RUN set -xe  \
         rclone \
     && pip3 install --upgrade pip \
     && pip3 install \
+        munch \
         oslo.rootwrap==6.2.0 \
         python-openstackclient==5.4.0 \
         python-swiftclient==3.10.1 \

tools/deployment/000-install-packages.sh

@@ -1,15 +1,58 @@
 #!/bin/bash
+
+# Copyright 2017 The Airship Authors.
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
 set -xe
+
 : "${INSTALL_PATH:="../"}"
-cd ${INSTALL_PATH}
+: "${OSH_COMMIT:="884a734833cb69a933416d7a04db25581097673c"}"
+: "${OSH_INFRA_COMMIT:="3d8935a536faaa87e084e7783ed53f4660a3a1f8"}"
+: "${CLONE_ARMADA:=false}"
+: "${CLONE_DECKHAND:=false}"
+: "${CLONE_SHIPYARD:=false}"
+: "${CLONE_PORTHOLE:=false}"
+: "${CLONE_MAAS:=false}"
+: "${CLONE_OSH:=true}"
+
+export INSTALL_PATH=${INSTALL_PATH}
+export CLONE_ARMADA=${CLONE_ARMADA}
+export CLONE_DECKHAND=${CLONE_DECKHAND}
+export CLONE_SHIPYARD=${CLONE_SHIPYARD}
+export CLONE_PORTHOLE=${CLONE_PORTHOLE}
+export CLONE_MAAS=${CLONE_MAAS}
+export CLONE_OSH=${CLONE_OSH}
+
+cd "${INSTALL_PATH}"
 
 # Clone dependencies
+rm -rf treasuremap
 rm -rf openstack-helm-infra
 rm -rf openstack-helm
-git clone https://opendev.org/openstack/openstack-helm-infra.git
+rm -rf maas
-git clone https://opendev.org/openstack/openstack-helm.git
+git clone https://opendev.org/airship/treasuremap.git
+pushd treasuremap
+git checkout v1.9
+
+
 # Install Packages
-bash -c "./openstack-helm-infra/tools/deployment/common/000-install-packages.sh"
+pwd
+bash -c "./tools/deployment/airskiff/developer/000-clone-dependencies.sh"
+
+find .. -maxdepth 1 -type d -print -exec sudo chmod -R o+rwx {} \;
 
 sudo apt-get update
 sudo apt-get install --no-install-recommends -y \

tools/deployment/001-setup-apparmor-profiles.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-bash -c "../openstack-helm-infra/tools/deployment/common/001-setup-apparmor-profiles.sh"
+bash -c "../treasuremap/tools/deployment/airskiff/developer/009-setup-apparmor.sh"

tools/deployment/002-build-charts.sh

@@ -1,15 +1,13 @@
 #!/bin/bash
 
-CURRENT_DIR="$(pwd)"
 : "${PORTHOLE_PATH:="../porthole"}"
 
 cd "${PORTHOLE_PATH}" || exit
-sudo echo 127.0.0.1 localhost /etc/hosts
+
 
 mkdir -p artifacts
 
-make lint
+make all
-make charts
 
 cd charts || exit
 for i in $(find  . -maxdepth 1  -name "*.tgz"  -print | sed -e 's/\-[0-9.]*\.tgz//'| cut -d / -f 2 | sort)

tools/deployment/003-deploy-k8s.sh

@@ -2,10 +2,10 @@
 set -x
 
 CURRENT_DIR="$(pwd)"
-: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
+: "${TREASUREMAP_PATH:="../treasuremap"}"
 
-cd "${OSH_INFRA_PATH}"
+cd "${TREASUREMAP_PATH}" || exit
-bash -c "./tools/deployment/common/005-deploy-k8s.sh"
+bash -c "./tools/deployment/airskiff/developer/010-deploy-k8s.sh"
 
 if [ -d /home/zuul ]
 then
@@ -13,3 +13,5 @@ then
     sudo chown -R zuul /home/zuul/.kube
 fi
 kubectl create namespace utility
+
+cd "${CURRENT_DIR}" || exit

tools/deployment/005-calicoctl-utility.sh

@@ -13,8 +13,12 @@
 
 set -xe
 namespace=utility
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_CALICOCTL_UTILITY:="$(./tools/deployment/get-values-overrides.sh calicoctl-utility)"}
+
 helm upgrade --install calicoctl-utility ./artifacts/calicoctl-utility.tgz --namespace=$namespace \
-    --set "images.tags.calicoctl_utility=quay.io/airshipit/porthole-calicoctl-utility:latest-${DISTRO}"
+    ${PORTHOLE_EXTRA_HELM_ARGS_CALICOCTL_UTILITY}
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/010-ceph-utility.sh

@@ -3,8 +3,15 @@ CURRENT_DIR="$(pwd)"
  : "${OSH_PATH:="../openstack-helm"}"
  : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
 
-cd "${OSH_PATH}"
+# remove previous loop devices if any
-bash -c "./tools/deployment/component/ceph/ceph.sh"
+
+if [ -f /etc/systemd/system/local-fs.target.wants/loops-setup.service ]
+then
+  sudo systemctl stop loops-setup
+fi
+
+cd "${OSH_PATH}" || exit
+sudo bash -c "./tools/deployment/component/ceph/ceph.sh"
 
 namespace="utility"
 : ${OSH_EXTRA_HELM_ARGS:=""}
@@ -52,8 +59,12 @@ helm upgrade --install ceph-utility-config ./ceph-provisioners \
 
 # Deploy Ceph-Utility
 cd ${CURRENT_DIR}
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_CEPH_UTILITY:="$(./tools/deployment/get-values-overrides.sh ceph-utility)"}
+
 helm upgrade --install ceph-utility ./artifacts/ceph-utility.tgz --namespace=$namespace \
-    --set "images.tags.ceph_utility=quay.io/airshipit/porthole-ceph-utility:latest-${DISTRO}"
+    ${PORTHOLE_EXTRA_HELM_ARGS_CEPH_UTILITY}
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/020-compute-utility.sh

@@ -14,8 +14,13 @@
 
 set -xe
 namespace="utility"
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_COMPUTE_UTILITY:="$(./tools/deployment/get-values-overrides.sh compute-utility)"}
+
 helm upgrade --install compute-utility ./artifacts/compute-utility.tgz --namespace=$namespace \
-    --set "images.tags.compute_utility=quay.io/airshipit/porthole-compute-utility:latest-${DISTRO}"
+    ${PORTHOLE_EXTRA_HELM_ARGS_COMPUTE_UTILITY}
+
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/030-etcdctl-utility.sh

@@ -13,8 +13,13 @@
 
 set -xe
 namespace="utility"
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_ETCDCTL_UTILITY:="$(./tools/deployment/get-values-overrides.sh etcdctl-utility)"}
+
 helm upgrade --install etcdctl-utility ./artifacts/etcdctl-utility.tgz --namespace=$namespace \
-    --set "images.tags.etcdctl_utility=quay.io/airshipit/porthole-etcdctl-utility:latest-${DISTRO}"
+    ${PORTHOLE_EXTRA_HELM_ARGS_ETCDCTL_UTILITY}
+
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/040-mysqlclient-utility.sh

@@ -35,7 +35,7 @@ export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../open
 : ${OSH_EXTRA_HELM_ARGS_MARIADB:="$(./tools/deployment/common/get-values-overrides.sh mariadb)"}
 
 #NOTE: Lint and package chart
-make -C ${HELM_CHART_ROOT_PATH} mariadb
+make -C "${HELM_CHART_ROOT_PATH}" mariadb
 
 #NOTE: Deploy command
 : ${OSH_EXTRA_HELM_ARGS:=""}
@@ -50,12 +50,16 @@ helm upgrade --install mariadb ${HELM_CHART_ROOT_PATH}/mariadb \
 ./tools/deployment/common/wait-for-pods.sh openstack
 
 # Deploy mysqlclient-utility
-cd ${CURRENT_DIR}
+cd "${CURRENT_DIR}"
 
 namespace="utility"
+
+export HELM_CHART_ROOT_PATH="${PORTHOLE_PATH:="../porthole/charts"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_MYSQLCLIENT_UTILITY:="$(./tools/deployment/get-values-overrides.sh mysqlclient-utility)"}
+
 helm upgrade --install mysqlclient-utility ./artifacts/mysqlclient-utility.tgz --namespace=$namespace \
-    --set "images.tags.mysqlclient_utility=quay.io/airshipit/porthole-mysqlclient-utility:latest-${DISTRO}" \
+    --set "conf.mariadb_backup_restore.enabled_namespaces=openstack" \
-    --set "conf.mariadb_backup_restore.enabled_namespaces=openstack"
+    ${PORTHOLE_EXTRA_HELM_ARGS_MYSQLCLIENT_UTILITY}
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/050-openstack-utility.sh

@@ -13,8 +13,12 @@
 
 set -xe
 namespace="utility"
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_OPENSTACK_UTILITY:="$(./tools/deployment/get-values-overrides.sh openstack-utility)"}
+
 helm upgrade --install openstack-utility ./artifacts/openstack-utility.tgz --namespace=$namespace \
-    --set "images.tags.openstack_utility=quay.io/airshipit/porthole-openstack-utility:latest-${DISTRO}"
+    ${PORTHOLE_EXTRA_HELM_ARGS_OPENSTACK_UTILITY}
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/060-postgresql-utility.sh

@@ -23,9 +23,14 @@ bash -c "./tools/deployment/common/020-ingress.sh"
 cd ${CURRENT_DIR}
 
 namespace="utility"
+
+export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
+: ${PORTHOLE_EXTRA_HELM_ARGS_POSTGRESQL_UTILITY:="$(./tools/deployment/get-values-overrides.sh postgresql-utility)"}
+
 helm upgrade --install postgresql-utility ./artifacts/postgresql-utility.tgz --namespace=$namespace \
-    --set "images.tags.postgresql_utility=quay.io/airshipit/porthole-postgresql-utility:latest-${DISTRO}" \
+    --set "conf.postgresql_backup_restore.enabled_namespaces=osh-infra" \
-    --set "conf.postgresql_backup_restore.enabled_namespaces=osh-infra"
+    ${PORTHOLE_EXTRA_HELM_ARGS_POSTGRESQL_UTILITY}
+
 
 # Wait for Deployment
 : "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

tools/deployment/env-variables.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+set -e
+
+export API_ADDR=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].addresses[0].ip')
+export API_PORT=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].ports[0].port')

tools/deployment/get-values-overrides.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+set -e
+
+HELM_CHART="$1"
+
+: "${HELM_CHART_ROOT_PATH:="../porthole/charts"}"
+: "${CONTAINER_DISTRO_NAME:="ubuntu"}"
+: "${CONTAINER_DISTRO_VERSION:="focal"}"
+: "${FEATURE_GATES:="images"}"
+PORTHOLE_FEATURE_MIX="${FEATURE_GATES},${CONTAINER_DISTRO_NAME}_${CONTAINER_DISTRO_VERSION},${CONTAINER_DISTRO_NAME}"
+
+function echoerr () {
+  echo "$@" 1>&2;
+}
+
+function generate_awk_exp_from_mask () {
+  local POSITION=1
+  for VALUE in $@; do
+    [ "${VALUE}" -eq 1 ] && echo -n "print \$${POSITION};"
+    POSITION=$((POSITION+1))
+  done
+  echo -e "\n"
+}
+
+function combination () {
+  POWER=$((2**$#))
+  BITS="$(awk "BEGIN { while (c++ < $#) printf \"0\" }")"
+  while [ "${POWER}" -gt 1 ];do
+    POWER=$((POWER-1))
+    BIN="$(bc <<< "obase=2; ${POWER}")"
+    MASK="$(echo "${BITS}" | sed -e "s/0\{${#BIN}\}$/$BIN/" | grep -o .)"
+    #NOTE: This line is odd, but written to support both BSD and GNU utils
+    awk -v ORS="-" "{$(generate_awk_exp_from_mask "$MASK")}" <<< "$@" | awk 1 | sed 's/-$//'
+  done
+}
+
+function override_file_args () {
+  OVERRIDE_ARGS=""
+  echoerr "We will attempt to use values-override files with the following paths:"
+  for FILE in $(combination ${1//,/ } | uniq | tac); do
+    FILE_PATH="${HELM_CHART_ROOT_PATH}/${HELM_CHART}/values_overrides/${FILE}.yaml"
+    if [ -f "${FILE_PATH}" ]; then
+      OVERRIDE_ARGS+=" --values=${FILE_PATH} "
+    fi
+      echoerr "${FILE_PATH}"
+  done
+  echo "${OVERRIDE_ARGS}"
+}
+
+echoerr "We are going to deploy the service ${HELM_CHART} using ${CONTAINER_DISTRO_NAME} (${CONTAINER_DISTRO_VERSION}) distribution containers."
+source ${HELM_CHART_ROOT_PATH}/../tools/deployment/env-variables.sh
+override_file_args "${PORTHOLE_FEATURE_MIX}"

tools/gate/deploy.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-./tools/deployment/000-install-packages.sh
-./tools/deployment/002-build-charts.sh
-./tools/deployment/003-deploy-k8s.sh
-./tools/deployment/004-make-images.sh
-./tools/deployment/005-calicoctl-utility.sh
-./tools/deployment/010-ceph-utility.sh
-./tools/deployment/020-compute-utility.sh
-./tools/deployment/030-etcdctl-utility.sh
-./tools/deployment/040-mysqlclient-utility.sh
-./tools/deployment/050-openstack-utility.sh
-./tools/deployment/060-postgresql-utility.sh
-sleep 60

tools/gate/playbooks/airship-porthole-gate-runner.yaml

@@ -19,6 +19,19 @@
     - clear-firewall
     - ensure-docker
   tasks:
+    - name: Install deps for tests
+      shell: |
+        set -x
+        ./tools/deployment/000-install-packages.sh
+        ./tools/deployment/002-build-charts.sh
+        ./tools/deployment/003-deploy-k8s.sh
+        ./tools/deployment/004-make-images.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ args.chdir }}"
+        executable: /bin/bash
+      environment:
+        DISTRO: "{{ distro_suffix | default('') }}"
+      become: True
     - name: Run gate scripts
       include_role:
         name: osh-run-script

tools/gate/playbooks/install-deps.yaml

@@ -1,33 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Combine several test tasks into a single playbook
-# to minimize Zuul node consumption
-
-- hosts: primary
-  roles:
-    - clear-firewall
-    - ensure-docker
-    - ensure-python
-    - ensure-tox
-  tasks:
-    - name: Install deps for tests
-      shell: |
-        set -x
-        env
-        ./tools/gate/deploy.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-        executable: /bin/bash
-      environment:
-        DISTRO: "{{ distro_suffix | default('') }}"
-      become: True

tools/gate/playbooks/make-functional-tests.yaml

@@ -12,6 +12,17 @@
 
 - hosts: primary
   tasks:
+
+    - name: Execute the make target for unit testing
+      block:
+        - make:
+            chdir: "{{ zuul.project.src_dir }}"
+            target: unit_tests
+          register: result
+          failed_when: result.failed
+        - debug:
+            var: result
+
     - name: Execute the make target for features testing
       block:
         - make:

tools/gate/playbooks/make-unit-tests.yaml

@@ -1,26 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: primary
-  tasks:
-    - name: Execute the make target for unit testing
-      block:
-        - make:
-            chdir: "{{ zuul.project.src_dir }}"
-            target: unit_tests
-          register: result
-          failed_when: result.failed
-        - debug:
-            var: result
-
-- name: Include collecting running logs
-  import_playbook: airship-porthole-collect-logs.yaml

tools/gate/playbooks/roles/osh-run-script/tasks/main.yaml

@@ -15,7 +15,7 @@
     set -xe;
     {{ gate_script_path }}
   args:
-    chdir: "{{ zuul.project.src_dir }}"
+    chdir: "{{ zuul.project.src_dir }}/{{ args.chdir }}"
   environment:
     zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
     OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.11.1-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {
@@ -31,9 +31,10 @@ function install_helm_binary {
   then
     TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
     curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
-    cd ${TMP_DIR}
+    pushd ${TMP_DIR}
     tar -xvzf helm.tar.gz
-    cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
+    cp "linux-amd64/helm" "${HELM}"
+    popd
   else
     echo "Cannot write to ${HELM}"
     exit -1
@@ -41,3 +42,4 @@ function install_helm_binary {
 }
 
 install_helm_binary
+${HELM} version

tools/helm_tk.sh

@@ -17,7 +17,7 @@
 set -eux
 
 HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"97ce6d7d8e9a090c748800d69a57bbd9af698b60"}
 
 TMP_DIR=$(mktemp -d)
 

zuul.d/base.yaml

@@ -35,10 +35,8 @@
         - airship-porthole-images-bionic-build-gate-mysqlclient-utility
         - airship-porthole-images-bionic-build-gate-openstack-utility
         - airship-porthole-images-bionic-build-gate-postgresql-utility
-        - airship-porthole-deploy-functional-unit-tests-ubuntu_focal
+        - airship-porthole-deploy-functional-tests-ubuntu_focal
-        - airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
+        - airship-porthole-deploy-functional-tests-ubuntu_bionic
-        - airship-porthole-deploy-functional-feature-tests-ubuntu_focal
-        - airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
 
     gate:
       jobs:
@@ -61,10 +59,8 @@
         - airship-porthole-images-bionic-build-gate-mysqlclient-utility
         - airship-porthole-images-bionic-build-gate-openstack-utility
         - airship-porthole-images-bionic-build-gate-postgresql-utility
-        - airship-porthole-deploy-functional-unit-tests-ubuntu_focal
+        - airship-porthole-deploy-functional-tests-ubuntu_focal
-        - airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
+        - airship-porthole-deploy-functional-tests-ubuntu_bionic
-        - airship-porthole-deploy-functional-feature-tests-ubuntu_focal
-        - airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
 
 
     post:
@@ -102,26 +98,46 @@
     parent: openstack-tox-py38
     description: Runs cover job on focal
     nodeset: airship-porthole-focal-single-node
-    pre-run: tools/gate/playbooks/install-deps.yaml
+    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
     timeout: 7200
     vars:
       distro_suffix: ubuntu_focal
+      gate_scripts:
+        - ./tools/deployment/005-calicoctl-utility.sh
+        - ./tools/deployment/010-ceph-utility.sh
+        - ./tools/deployment/020-compute-utility.sh
+        - ./tools/deployment/030-etcdctl-utility.sh
+        - ./tools/deployment/040-mysqlclient-utility.sh
+        - ./tools/deployment/050-openstack-utility.sh
+        - ./tools/deployment/060-postgresql-utility.sh
+      args:
+        chdir: ../porthole
 
 - job:
     name: airship-porthole-openstack-tox-cover
     parent: openstack-tox-cover
     description: Runs cover job on focal
     nodeset: airship-porthole-focal-single-node
-    pre-run: tools/gate/playbooks/install-deps.yaml
+    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
     timeout: 7200
     vars:
       distro_suffix: ubuntu_focal
+      gate_scripts:
+        - ./tools/deployment/005-calicoctl-utility.sh
+        - ./tools/deployment/010-ceph-utility.sh
+        - ./tools/deployment/020-compute-utility.sh
+        - ./tools/deployment/030-etcdctl-utility.sh
+        - ./tools/deployment/040-mysqlclient-utility.sh
+        - ./tools/deployment/050-openstack-utility.sh
+        - ./tools/deployment/060-postgresql-utility.sh
+      args:
+        chdir: ../porthole
 
 - job:
     name: airship-porthole-images
     abstract: true
-    timeout: 1800
+    timeout: 7200
-    post-timeout: 1800
+    post-timeout: 7200
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-porthole-focal-single-node
     vars:
@@ -142,66 +158,58 @@
     timeout: 300
     nodeset: airship-porthole-focal-single-node
 
-
 - job:
-    name: airship-porthole-deploy-functional-unit-tests-ubuntu_focal
+    name: airship-porthole-deploy-functional-tests-ubuntu_focal
     description: |
-       Deploys all UC's and Executes unit and feature tests.
+       Deploys all UC's and Executes functional tests on focal.
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
     nodeset: airship-porthole-focal-single-node
     timeout: 7200
-    run:  tools/gate/playbooks/make-unit-tests.yaml
+    run:
+      - tools/gate/playbooks/airship-porthole-gate-runner.yaml
+      - tools/gate/playbooks/make-functional-tests.yaml
     vars:
       distro_suffix: ubuntu_focal
       gate_scripts:
-         - ./tools/gate/deploy.sh
+        - ./tools/deployment/005-calicoctl-utility.sh
+        - ./tools/deployment/010-ceph-utility.sh
+        - ./tools/deployment/020-compute-utility.sh
+        - ./tools/deployment/030-etcdctl-utility.sh
+        - ./tools/deployment/040-mysqlclient-utility.sh
+        - ./tools/deployment/050-openstack-utility.sh
+        - ./tools/deployment/060-postgresql-utility.sh
+      osh_params:
+        openstack_release: yoga
+        container_distro_name: ubuntu
+        container_distro_version: focal
       args:
-        chdir: "{{ zuul.project.src_dir }}"
+        chdir: ../porthole
+
 
 - job:
-    name: airship-porthole-deploy-functional-feature-tests-ubuntu_focal
+    name: airship-porthole-deploy-functional-tests-ubuntu_bionic
     description: |
-       Deploys all UC's and Executesfeature tests.
+       Deploys all UC's and Executes functions tests on bionic.
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
     nodeset: airship-porthole-focal-single-node
     timeout: 7200
-    run: tools/gate/playbooks/make-feature-tests.yaml
+    run:
-    vars:
+      - tools/gate/playbooks/airship-porthole-gate-runner.yaml
-      distro_suffix: ubuntu_focal
+      - tools/gate/playbooks/make-functional-tests.yaml
-      gate_scripts:
-         - ./tools/gate/deploy.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
-
-- job:
-    name: airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
-    description: |
-       Deploys all UC's and Executes unit  tests.
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    nodeset: airship-porthole-focal-single-node
-    timeout: 7200
-    run: tools/gate/playbooks/make-unit-tests.yaml
     vars:
       distro_suffix: ubuntu_bionic
       gate_scripts:
-         - ./tools/gate/deploy.sh
+        - ./tools/deployment/005-calicoctl-utility.sh
+        - ./tools/deployment/010-ceph-utility.sh
+        - ./tools/deployment/020-compute-utility.sh
+        - ./tools/deployment/030-etcdctl-utility.sh
+        - ./tools/deployment/040-mysqlclient-utility.sh
+        - ./tools/deployment/050-openstack-utility.sh
+        - ./tools/deployment/060-postgresql-utility.sh
+      osh_params:
+        openstack_release: yoga
+        container_distro_name: ubuntu
+        container_distro_version: bionic
       args:
-        chdir: "{{ zuul.project.src_dir }}"
+        chdir: ../porthole
-
-- job:
-    name: airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
-    description: |
-       Deploys all UC's and Executes feature tests.
-    pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
-    nodeset: airship-porthole-focal-single-node
-    timeout: 7200
-    run: tools/gate/playbooks/make-feature-tests.yaml
-    vars:
-      distro_suffix: ubuntu_bionic
-      gate_scripts:
-         - ./tools/gate/deploy.sh
-      args:
-        chdir: "{{ zuul.project.src_dir }}"
 
 - secret:
     name: quay_credentials