airship/porthole
Code Issues Proposed changes

Porthole upgrades

Browse Source 
This PS delivers the following changes:

- adds values_overrides folders into utility binaries charts for
  flexible testing the charts and images;
- values_overrides added to .helmignore files for every chart;
- added munch python package into bionic images for compartibility with
  newer openstack client dependences;
- improved Makefile clean section;
- uses treasuremap v1.9 deploy-k8s.sh script to deploy test k8s cluster;
- adds appropriate changes to deploy k8s script, setup apparmor;
- porthole utility charts deployment script have been enriched with and
  ability to control additional values from values_overrides folders of
  each chart;
- added env-variables.sh and get-values-overrides.sh to power up the
  changes describes in previous point;
- removed deploy.sh script for clarity of actions in playbooks;
- improved airship-porthole-gate-runner playbook;
- removed install-deps playbook that was used to pre-setup zuul gate;
- united make-unit-tests and make-feature-tests playbooks into one
  make-functional tests playbook;
- helm version has been bumped up to 3.11.1;
- improved helm_tk.sh script in order to solve issues with multiple
  charts build process;
- bumped up version of stable helm-toolkit used for charts build;
- airship-porthole-gate-runner playbook is now used as pre-run script
  for tox profiles gates and as main run playbook for functional tests
  gate;

Change-Id: I202c573ff85ed2b80191057b537a65a5f5d4600e
This commit is contained in:
Sergiy Markin 2023-06-17 19:12:59 +00:00
parent 5ead2930ae
commit 04a8cb59b5
49 changed files with 369 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -69,9 +69,9 @@ releasenotes/build
**/*.tgz
**/_partials.tpl
**/_globals.tpl
/charts/deps/*/
/charts/deps/*
# Gate and Check Logs
logs/
tmp/

5
Makefile
View File

@ -22,7 +22,7 @@ IMAGE_NAME ?= $@
IMAGE_TAG ?= latest
BUILD_DIR := $(shell mktemp -d)
HELM ?= $(BUILD_DIR)/helm
CHARTS := $(patsubst charts/%/.,%,$(wildcard charts/*/.))
CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
IMAGES := $(addprefix porthole-,$(patsubst images/%/.,%,$(wildcard images/*/.)))
PROXY ?= http://proxy.foo.com:8000
NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local
@ -125,6 +125,9 @@ clean:
rm -f charts/*.tgz
rm -f charts/*/requirements.lock
rm -rf charts/*/charts
rm -rf charts/deps/*
rm -rf .tox
rm -rf cover
run_images:
@echo "Not implemented." >&2; exit 2

1
charts/calicoctl-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/calicoctl-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
calicoctl_utility: quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_bionic
...

5
charts/calicoctl-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
calicoctl_utility: quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_focal
...

1
charts/ceph-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/ceph-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
ceph_utility: quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_bionic
...

5
charts/ceph-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
ceph_utility: quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_focal
...

1
charts/compute-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/compute-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
compute_utility: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_bionic
...

5
charts/compute-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
compute_utility: quay.io/airshipit/porthole-compute-utility:latest-ubuntu_focal
...

0
charts/deps/.gitkeep Normal file
View File

1
charts/etcdctl-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/etcdctl-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
etcdctl_utility: quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_bionic
...

5
charts/etcdctl-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
etcdctl_utility: quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_focal
...

1
charts/mysqlclient-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/mysqlclient-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
mysqlclient_utility: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic
...

5
charts/mysqlclient-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
mysqlclient_utility: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal
...

1
charts/openstack-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/openstack-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
openstack_utility: quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_bionic
...

5
charts/openstack-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
openstack_utility: quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_focal
...

1
charts/postgresql-utility/.helmignore Normal file
View File

@ -0,0 +1 @@
values_overrides

5
charts/postgresql-utility/values_overrides/images-ubuntu_bionic.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
postgresql_utility: quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic
...

5
charts/postgresql-utility/values_overrides/images-ubuntu_focal.yaml Normal file
View File

@ -0,0 +1,5 @@
---
images:
tags:
postgresql_utility: quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_focal
...

1
images/etcdctl-utility/Dockerfile.ubuntu_bionic
View File

@ -36,6 +36,7 @@ RUN set -xe \
rclone \
&& pip3 install --upgrade pip \
&& pip3 install \
munch \
oslo.rootwrap==6.2.0 \
python-openstackclient==5.4.0 \
python-swiftclient==3.10.1 \

3
images/mysqlclient-utility/Dockerfile.ubuntu_bionic
View File

@ -36,6 +36,7 @@ RUN set -xe \
rclone \
&& pip3 install --upgrade pip \
&& pip3 install \
munch \
oslo.rootwrap==6.2.0 \
python-openstackclient==5.4.0 \
python-swiftclient==3.10.1 \
@ -43,7 +44,7 @@ RUN set -xe \
&& apt-get clean -y \
&& curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \
| tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \
&& curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \
&& curl --silent -L https://downloads.percona.com/downloads/percona-toolkit/3.5.1/binary/tarball/percona-toolkit-3.5.1_x86_64.tar.gz \
| tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \
&& rm -rf \
/tmp/* \

1
images/postgresql-utility/Dockerfile.ubuntu_bionic
View File

@ -34,6 +34,7 @@ RUN set -xe \
rclone \
&& pip3 install --upgrade pip \
&& pip3 install \
munch \
oslo.rootwrap==6.2.0 \
python-openstackclient==5.4.0 \
python-swiftclient==3.10.1 \

51
tools/deployment/000-install-packages.sh
View File

@ -1,15 +1,58 @@
#!/bin/bash
# Copyright 2017 The Airship Authors.
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
: "${INSTALL_PATH:="../"}"
cd ${INSTALL_PATH}
: "${OSH_COMMIT:="884a734833cb69a933416d7a04db25581097673c"}"
: "${OSH_INFRA_COMMIT:="3d8935a536faaa87e084e7783ed53f4660a3a1f8"}"
: "${CLONE_ARMADA:=false}"
: "${CLONE_DECKHAND:=false}"
: "${CLONE_SHIPYARD:=false}"
: "${CLONE_PORTHOLE:=false}"
: "${CLONE_MAAS:=false}"
: "${CLONE_OSH:=true}"
export INSTALL_PATH=${INSTALL_PATH}
export CLONE_ARMADA=${CLONE_ARMADA}
export CLONE_DECKHAND=${CLONE_DECKHAND}
export CLONE_SHIPYARD=${CLONE_SHIPYARD}
export CLONE_PORTHOLE=${CLONE_PORTHOLE}
export CLONE_MAAS=${CLONE_MAAS}
export CLONE_OSH=${CLONE_OSH}
cd "${INSTALL_PATH}"
# Clone dependencies
rm -rf treasuremap
rm -rf openstack-helm-infra
rm -rf openstack-helm
git clone https://opendev.org/openstack/openstack-helm-infra.git
git clone https://opendev.org/openstack/openstack-helm.git
rm -rf maas
git clone https://opendev.org/airship/treasuremap.git
pushd treasuremap
git checkout v1.9
# Install Packages
bash -c "./openstack-helm-infra/tools/deployment/common/000-install-packages.sh"
pwd
bash -c "./tools/deployment/airskiff/developer/000-clone-dependencies.sh"
find .. -maxdepth 1 -type d -print -exec sudo chmod -R o+rwx {} \;
sudo apt-get update
sudo apt-get install --no-install-recommends -y \

2
tools/deployment/001-setup-apparmor-profiles.sh
View File

@ -1,2 +1,2 @@
#!/bin/bash
bash -c "../openstack-helm-infra/tools/deployment/common/001-setup-apparmor-profiles.sh"
bash -c "../treasuremap/tools/deployment/airskiff/developer/009-setup-apparmor.sh"

6
tools/deployment/002-build-charts.sh
View File

@ -1,15 +1,13 @@
#!/bin/bash
CURRENT_DIR="$(pwd)"
: "${PORTHOLE_PATH:="../porthole"}"
cd "${PORTHOLE_PATH}" || exit
sudo echo 127.0.0.1 localhost /etc/hosts
mkdir -p artifacts
make lint
make charts
make all
cd charts || exit
for i in $(find . -maxdepth 1 -name "*.tgz" -print | sed -e 's/\-[0-9.]*\.tgz//'| cut -d / -f 2 | sort)

8
tools/deployment/003-deploy-k8s.sh
View File

@ -2,10 +2,10 @@
set -x
CURRENT_DIR="$(pwd)"
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
: "${TREASUREMAP_PATH:="../treasuremap"}"
cd "${OSH_INFRA_PATH}"
bash -c "./tools/deployment/common/005-deploy-k8s.sh"
cd "${TREASUREMAP_PATH}" || exit
bash -c "./tools/deployment/airskiff/developer/010-deploy-k8s.sh"
if [ -d /home/zuul ]
then
@ -13,3 +13,5 @@ then
sudo chown -R zuul /home/zuul/.kube
fi
kubectl create namespace utility
cd "${CURRENT_DIR}" || exit

6
tools/deployment/005-calicoctl-utility.sh
View File

@ -13,8 +13,12 @@
set -xe
namespace=utility
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_CALICOCTL_UTILITY:="$(./tools/deployment/get-values-overrides.sh calicoctl-utility)"}
helm upgrade --install calicoctl-utility ./artifacts/calicoctl-utility.tgz --namespace=$namespace \
--set "images.tags.calicoctl_utility=quay.io/airshipit/porthole-calicoctl-utility:latest-${DISTRO}"
${PORTHOLE_EXTRA_HELM_ARGS_CALICOCTL_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

17
tools/deployment/010-ceph-utility.sh
View File

@ -3,8 +3,15 @@ CURRENT_DIR="$(pwd)"
: "${OSH_PATH:="../openstack-helm"}"
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
cd "${OSH_PATH}"
bash -c "./tools/deployment/component/ceph/ceph.sh"
# remove previous loop devices if any
if [ -f /etc/systemd/system/local-fs.target.wants/loops-setup.service ]
then
sudo systemctl stop loops-setup
fi
cd "${OSH_PATH}" || exit
sudo bash -c "./tools/deployment/component/ceph/ceph.sh"
namespace="utility"
: ${OSH_EXTRA_HELM_ARGS:=""}
@ -52,8 +59,12 @@ helm upgrade --install ceph-utility-config ./ceph-provisioners \
# Deploy Ceph-Utility
cd ${CURRENT_DIR}
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_CEPH_UTILITY:="$(./tools/deployment/get-values-overrides.sh ceph-utility)"}
helm upgrade --install ceph-utility ./artifacts/ceph-utility.tgz --namespace=$namespace \
--set "images.tags.ceph_utility=quay.io/airshipit/porthole-ceph-utility:latest-${DISTRO}"
${PORTHOLE_EXTRA_HELM_ARGS_CEPH_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

7
tools/deployment/020-compute-utility.sh
View File

@ -14,8 +14,13 @@
set -xe
namespace="utility"
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_COMPUTE_UTILITY:="$(./tools/deployment/get-values-overrides.sh compute-utility)"}
helm upgrade --install compute-utility ./artifacts/compute-utility.tgz --namespace=$namespace \
--set "images.tags.compute_utility=quay.io/airshipit/porthole-compute-utility:latest-${DISTRO}"
${PORTHOLE_EXTRA_HELM_ARGS_COMPUTE_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

7
tools/deployment/030-etcdctl-utility.sh
View File

@ -13,8 +13,13 @@
set -xe
namespace="utility"
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_ETCDCTL_UTILITY:="$(./tools/deployment/get-values-overrides.sh etcdctl-utility)"}
helm upgrade --install etcdctl-utility ./artifacts/etcdctl-utility.tgz --namespace=$namespace \
--set "images.tags.etcdctl_utility=quay.io/airshipit/porthole-etcdctl-utility:latest-${DISTRO}"
${PORTHOLE_EXTRA_HELM_ARGS_ETCDCTL_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

12
tools/deployment/040-mysqlclient-utility.sh
View File

@ -35,7 +35,7 @@ export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${OSH_INFRA_PATH:="../open
: ${OSH_EXTRA_HELM_ARGS_MARIADB:="$(./tools/deployment/common/get-values-overrides.sh mariadb)"}
#NOTE: Lint and package chart
make -C ${HELM_CHART_ROOT_PATH} mariadb
make -C "${HELM_CHART_ROOT_PATH}" mariadb
#NOTE: Deploy command
: ${OSH_EXTRA_HELM_ARGS:=""}
@ -50,12 +50,16 @@ helm upgrade --install mariadb ${HELM_CHART_ROOT_PATH}/mariadb \
./tools/deployment/common/wait-for-pods.sh openstack
# Deploy mysqlclient-utility
cd ${CURRENT_DIR}
cd "${CURRENT_DIR}"
namespace="utility"
export HELM_CHART_ROOT_PATH="${PORTHOLE_PATH:="../porthole/charts"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_MYSQLCLIENT_UTILITY:="$(./tools/deployment/get-values-overrides.sh mysqlclient-utility)"}
helm upgrade --install mysqlclient-utility ./artifacts/mysqlclient-utility.tgz --namespace=$namespace \
--set "images.tags.mysqlclient_utility=quay.io/airshipit/porthole-mysqlclient-utility:latest-${DISTRO}" \
--set "conf.mariadb_backup_restore.enabled_namespaces=openstack"
--set "conf.mariadb_backup_restore.enabled_namespaces=openstack" \
${PORTHOLE_EXTRA_HELM_ARGS_MYSQLCLIENT_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

6
tools/deployment/050-openstack-utility.sh
View File

@ -13,8 +13,12 @@
set -xe
namespace="utility"
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_OPENSTACK_UTILITY:="$(./tools/deployment/get-values-overrides.sh openstack-utility)"}
helm upgrade --install openstack-utility ./artifacts/openstack-utility.tgz --namespace=$namespace \
--set "images.tags.openstack_utility=quay.io/airshipit/porthole-openstack-utility:latest-${DISTRO}"
${PORTHOLE_EXTRA_HELM_ARGS_OPENSTACK_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

9
tools/deployment/060-postgresql-utility.sh
View File

@ -23,9 +23,14 @@ bash -c "./tools/deployment/common/020-ingress.sh"
cd ${CURRENT_DIR}
namespace="utility"
export HELM_CHART_ROOT_PATH="${HELM_CHART_ROOT_PATH:="${PORTHOLE_PATH:="../porthole/charts"}"}"
: ${PORTHOLE_EXTRA_HELM_ARGS_POSTGRESQL_UTILITY:="$(./tools/deployment/get-values-overrides.sh postgresql-utility)"}
helm upgrade --install postgresql-utility ./artifacts/postgresql-utility.tgz --namespace=$namespace \
--set "images.tags.postgresql_utility=quay.io/airshipit/porthole-postgresql-utility:latest-${DISTRO}" \
--set "conf.postgresql_backup_restore.enabled_namespaces=osh-infra"
--set "conf.postgresql_backup_restore.enabled_namespaces=osh-infra" \
${PORTHOLE_EXTRA_HELM_ARGS_POSTGRESQL_UTILITY}
# Wait for Deployment
: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"

17
tools/deployment/env-variables.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
export API_ADDR=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].addresses[0].ip')
export API_PORT=$(kubectl get endpoints kubernetes -o json | jq -r '.subsets[0].ports[0].port')

65
tools/deployment/get-values-overrides.sh Executable file
View File

@ -0,0 +1,65 @@
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
HELM_CHART="$1"
: "${HELM_CHART_ROOT_PATH:="../porthole/charts"}"
: "${CONTAINER_DISTRO_NAME:="ubuntu"}"
: "${CONTAINER_DISTRO_VERSION:="focal"}"
: "${FEATURE_GATES:="images"}"
PORTHOLE_FEATURE_MIX="${FEATURE_GATES},${CONTAINER_DISTRO_NAME}_${CONTAINER_DISTRO_VERSION},${CONTAINER_DISTRO_NAME}"
function echoerr () {
echo "$@" 1>&2;
}
function generate_awk_exp_from_mask () {
local POSITION=1
for VALUE in $@; do
[ "${VALUE}" -eq 1 ] && echo -n "print \$${POSITION};"
POSITION=$((POSITION+1))
done
echo -e "\n"
}
function combination () {
POWER=$((2**$#))
BITS="$(awk "BEGIN { while (c++ < $#) printf \"0\" }")"
while [ "${POWER}" -gt 1 ];do
POWER=$((POWER-1))
BIN="$(bc <<< "obase=2; ${POWER}")"
MASK="$(echo "${BITS}" | sed -e "s/0\{${#BIN}\}$/$BIN/" | grep -o .)"
#NOTE: This line is odd, but written to support both BSD and GNU utils
awk -v ORS="-" "{$(generate_awk_exp_from_mask "$MASK")}" <<< "$@" | awk 1 | sed 's/-$//'
done
}
function override_file_args () {
OVERRIDE_ARGS=""
echoerr "We will attempt to use values-override files with the following paths:"
for FILE in $(combination ${1//,/ } | uniq | tac); do
FILE_PATH="${HELM_CHART_ROOT_PATH}/${HELM_CHART}/values_overrides/${FILE}.yaml"
if [ -f "${FILE_PATH}" ]; then
OVERRIDE_ARGS+=" --values=${FILE_PATH} "
fi
echoerr "${FILE_PATH}"
done
echo "${OVERRIDE_ARGS}"
}
echoerr "We are going to deploy the service ${HELM_CHART} using ${CONTAINER_DISTRO_NAME} (${CONTAINER_DISTRO_VERSION}) distribution containers."
source ${HELM_CHART_ROOT_PATH}/../tools/deployment/env-variables.sh
override_file_args "${PORTHOLE_FEATURE_MIX}"

16
tools/gate/deploy.sh
View File

@ -1,16 +0,0 @@
#!/bin/bash
set -ex
./tools/deployment/000-install-packages.sh
./tools/deployment/002-build-charts.sh
./tools/deployment/003-deploy-k8s.sh
./tools/deployment/004-make-images.sh
./tools/deployment/005-calicoctl-utility.sh
./tools/deployment/010-ceph-utility.sh
./tools/deployment/020-compute-utility.sh
./tools/deployment/030-etcdctl-utility.sh
./tools/deployment/040-mysqlclient-utility.sh
./tools/deployment/050-openstack-utility.sh
./tools/deployment/060-postgresql-utility.sh
sleep 60

13
tools/gate/playbooks/airship-porthole-gate-runner.yaml
View File

@ -19,6 +19,19 @@
- clear-firewall
- ensure-docker
tasks:
- name: Install deps for tests
shell: |
set -x
./tools/deployment/000-install-packages.sh
./tools/deployment/002-build-charts.sh
./tools/deployment/003-deploy-k8s.sh
./tools/deployment/004-make-images.sh
args:
chdir: "{{ zuul.project.src_dir }}/{{ args.chdir }}"
executable: /bin/bash
environment:
DISTRO: "{{ distro_suffix | default('') }}"
become: True
- name: Run gate scripts
include_role:
name: osh-run-script

33
tools/gate/playbooks/install-deps.yaml
View File

@ -1,33 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Combine several test tasks into a single playbook
# to minimize Zuul node consumption
- hosts: primary
roles:
- clear-firewall
- ensure-docker
- ensure-python
- ensure-tox
tasks:
- name: Install deps for tests
shell: |
set -x
env
./tools/gate/deploy.sh
args:
chdir: "{{ zuul.project.src_dir }}"
executable: /bin/bash
environment:
DISTRO: "{{ distro_suffix | default('') }}"
become: True

11
tools/gate/playbooks/make-feature-tests.yaml → tools/gate/playbooks/make-functional-tests.yaml
View File

@ -12,6 +12,17 @@
- hosts: primary
tasks:
- name: Execute the make target for unit testing
block:
- make:
chdir: "{{ zuul.project.src_dir }}"
target: unit_tests
register: result
failed_when: result.failed
- debug:
var: result
- name: Execute the make target for features testing
block:
- make:

26
tools/gate/playbooks/make-unit-tests.yaml
View File

@ -1,26 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: primary
tasks:
- name: Execute the make target for unit testing
block:
- make:
chdir: "{{ zuul.project.src_dir }}"
target: unit_tests
register: result
failed_when: result.failed
- debug:
var: result
- name: Include collecting running logs
import_playbook: airship-porthole-collect-logs.yaml

2
tools/gate/playbooks/roles/osh-run-script/tasks/main.yaml
View File

@ -15,7 +15,7 @@
set -xe;
{{ gate_script_path }}
args:
chdir: "{{ zuul.project.src_dir }}"
chdir: "{{ zuul.project.src_dir }}/{{ args.chdir }}"
environment:
zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"

8
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.11.1-linux-amd64.tar.gz"}
function install_helm_binary {
@ -31,9 +31,10 @@ function install_helm_binary {
then
TMP_DIR=${BUILD_DIR:-$(mktemp -d)}
curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}"
cd ${TMP_DIR}
pushd ${TMP_DIR}
tar -xvzf helm.tar.gz
cp "${TMP_DIR}/linux-amd64/helm" "${HELM}"
cp "linux-amd64/helm" "${HELM}"
popd
else
echo "Cannot write to ${HELM}"
exit -1
@ -41,3 +42,4 @@ function install_helm_binary {
}
install_helm_binary
${HELM} version

2
tools/helm_tk.sh
View File

@ -17,7 +17,7 @@
set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"97ce6d7d8e9a090c748800d69a57bbd9af698b60"}
TMP_DIR=$(mktemp -d)

118
zuul.d/base.yaml
View File

@ -35,10 +35,8 @@
- airship-porthole-images-bionic-build-gate-mysqlclient-utility
- airship-porthole-images-bionic-build-gate-openstack-utility
- airship-porthole-images-bionic-build-gate-postgresql-utility
- airship-porthole-deploy-functional-unit-tests-ubuntu_focal
- airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
- airship-porthole-deploy-functional-feature-tests-ubuntu_focal
- airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
- airship-porthole-deploy-functional-tests-ubuntu_focal
- airship-porthole-deploy-functional-tests-ubuntu_bionic
gate:
jobs:
@ -61,10 +59,8 @@
- airship-porthole-images-bionic-build-gate-mysqlclient-utility
- airship-porthole-images-bionic-build-gate-openstack-utility
- airship-porthole-images-bionic-build-gate-postgresql-utility
- airship-porthole-deploy-functional-unit-tests-ubuntu_focal
- airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
- airship-porthole-deploy-functional-feature-tests-ubuntu_focal
- airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
- airship-porthole-deploy-functional-tests-ubuntu_focal
- airship-porthole-deploy-functional-tests-ubuntu_bionic
post:
@ -102,26 +98,46 @@
parent: openstack-tox-py38
description: Runs cover job on focal
nodeset: airship-porthole-focal-single-node
pre-run: tools/gate/playbooks/install-deps.yaml
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
timeout: 7200
vars:
distro_suffix: ubuntu_focal
gate_scripts:
- ./tools/deployment/005-calicoctl-utility.sh
- ./tools/deployment/010-ceph-utility.sh
- ./tools/deployment/020-compute-utility.sh
- ./tools/deployment/030-etcdctl-utility.sh
- ./tools/deployment/040-mysqlclient-utility.sh
- ./tools/deployment/050-openstack-utility.sh
- ./tools/deployment/060-postgresql-utility.sh
args:
chdir: ../porthole
- job:
name: airship-porthole-openstack-tox-cover
parent: openstack-tox-cover
description: Runs cover job on focal
nodeset: airship-porthole-focal-single-node
pre-run: tools/gate/playbooks/install-deps.yaml
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
timeout: 7200
vars:
distro_suffix: ubuntu_focal
gate_scripts:
- ./tools/deployment/005-calicoctl-utility.sh
- ./tools/deployment/010-ceph-utility.sh
- ./tools/deployment/020-compute-utility.sh
- ./tools/deployment/030-etcdctl-utility.sh
- ./tools/deployment/040-mysqlclient-utility.sh
- ./tools/deployment/050-openstack-utility.sh
- ./tools/deployment/060-postgresql-utility.sh
args:
chdir: ../porthole
- job:
name: airship-porthole-images
abstract: true
timeout: 1800
post-timeout: 1800
timeout: 7200
post-timeout: 7200
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: airship-porthole-focal-single-node
vars:
@ -142,66 +158,58 @@
timeout: 300
nodeset: airship-porthole-focal-single-node
- job:
name: airship-porthole-deploy-functional-unit-tests-ubuntu_focal
name: airship-porthole-deploy-functional-tests-ubuntu_focal
description: |
Deploys all UC's and Executes unit and feature tests.
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
Deploys all UC's and Executes functional tests on focal.
nodeset: airship-porthole-focal-single-node
timeout: 7200
run: tools/gate/playbooks/make-unit-tests.yaml
run:
- tools/gate/playbooks/airship-porthole-gate-runner.yaml
- tools/gate/playbooks/make-functional-tests.yaml
vars:
distro_suffix: ubuntu_focal
gate_scripts:
- ./tools/gate/deploy.sh
- ./tools/deployment/005-calicoctl-utility.sh
- ./tools/deployment/010-ceph-utility.sh
- ./tools/deployment/020-compute-utility.sh
- ./tools/deployment/030-etcdctl-utility.sh
- ./tools/deployment/040-mysqlclient-utility.sh
- ./tools/deployment/050-openstack-utility.sh
- ./tools/deployment/060-postgresql-utility.sh
osh_params:
openstack_release: yoga
container_distro_name: ubuntu
container_distro_version: focal
args:
chdir: "{{ zuul.project.src_dir }}"
chdir: ../porthole
- job:
name: airship-porthole-deploy-functional-feature-tests-ubuntu_focal
name: airship-porthole-deploy-functional-tests-ubuntu_bionic
description: |
Deploys all UC's and Executesfeature tests.
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
Deploys all UC's and Executes functions tests on bionic.
nodeset: airship-porthole-focal-single-node
timeout: 7200
run: tools/gate/playbooks/make-feature-tests.yaml
vars:
distro_suffix: ubuntu_focal
gate_scripts:
- ./tools/gate/deploy.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- job:
name: airship-porthole-deploy-functional-unit-tests-ubuntu_bionic
description: |
Deploys all UC's and Executes unit tests.
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
nodeset: airship-porthole-focal-single-node
timeout: 7200
run: tools/gate/playbooks/make-unit-tests.yaml
run:
- tools/gate/playbooks/airship-porthole-gate-runner.yaml
- tools/gate/playbooks/make-functional-tests.yaml
vars:
distro_suffix: ubuntu_bionic
gate_scripts:
- ./tools/gate/deploy.sh
- ./tools/deployment/005-calicoctl-utility.sh
- ./tools/deployment/010-ceph-utility.sh
- ./tools/deployment/020-compute-utility.sh
- ./tools/deployment/030-etcdctl-utility.sh
- ./tools/deployment/040-mysqlclient-utility.sh
- ./tools/deployment/050-openstack-utility.sh
- ./tools/deployment/060-postgresql-utility.sh
osh_params:
openstack_release: yoga
container_distro_name: ubuntu
container_distro_version: bionic
args:
chdir: "{{ zuul.project.src_dir }}"
- job:
name: airship-porthole-deploy-functional-feature-tests-ubuntu_bionic
description: |
Deploys all UC's and Executes feature tests.
pre-run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
nodeset: airship-porthole-focal-single-node
timeout: 7200
run: tools/gate/playbooks/make-feature-tests.yaml
vars:
distro_suffix: ubuntu_bionic
gate_scripts:
- ./tools/gate/deploy.sh
args:
chdir: "{{ zuul.project.src_dir }}"
chdir: ../porthole
- secret:
name: quay_credentials