airship/armada
Code Issues Proposed changes

Update armada to focal

Browse Source 
Update kubernetes client to v26.1.0
Updating armada to focal base image
Remove xenial and opensuse dockerfiles
Update tox python from py35 to py38
Add apparmor for docker build
Uplift HTK chart version 0.2.52
Bumping up some python dependencies to get in sync with shipyard
Added clear-firewall role for airskiff-deploy playbook

Change-Id: If06a3f60466702d05a21c24a7cb8041bed41507a
This commit is contained in:
SPEARS, DUSTIN (ds443n) 2023-02-13 10:11:12 -05:00
parent 099de8aaf4
commit 0e9d828fe6
14 changed files with 104 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
.zuul.yaml
View File

@ -13,30 +13,28 @@
- project:
templates:
- openstack-cover-jobs
- openstack-python36-jobs
- openstack-python38-jobs
check:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-chart-build-latest-htk
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_xenial
- armada-docker-build-gate-opensuse
# - armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy
gate:
jobs:
- openstack-tox-docs
- openstack-tox-pep8
- armada-chart-build-gate
- armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_xenial
- armada-docker-build-gate-opensuse
# - armada-docker-build-gate-ubuntu_bionic
- armada-docker-build-gate-ubuntu_focal
- armada-airskiff-deploy
post:
jobs:
- armada-docker-publish-ubuntu_bionic
- armada-docker-publish-ubuntu_xenial
- armada-docker-publish-opensuse
- armada-docker-build-gate-ubuntu_focal
- armada-docker-publish-ubuntu_focal
- armada-upload-git-mirror
promote:
jobs:
@ -48,13 +46,19 @@
- name: primary
label: ubuntu-bionic
- nodeset:
name: armada-single-node-focal
nodes:
- name: primary
label: ubuntu-focal
- job:
name: armada-chart-build-gate
description: |
Builds charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: armada-single-node
nodeset: armada-single-node-focal
- job:
name: armada-chart-build-latest-htk
@ -63,49 +67,37 @@
timeout: 900
voting: false
run: tools/gate/playbooks/build-charts.yaml
nodeset: armada-single-node
nodeset: armada-single-node-focal
vars:
HTK_COMMIT: master
- job:
name: armada-docker-build-gate-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
vars:
publish: false
distro: ubuntu_bionic
tags:
dynamic:
patch_set: true
# - job:
# name: armada-docker-build-gate-ubuntu_bionic
# timeout: 1800
# run: tools/gate/playbooks/docker-image-build.yaml
# nodeset: armada-single-node
# vars:
# publish: false
# distro: ubuntu_bionic
# tags:
# dynamic:
# patch_set: true
- job:
name: armada-docker-build-gate-ubuntu_xenial
name: armada-docker-build-gate-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
nodeset: armada-single-node-focal
vars:
publish: false
distro: ubuntu_xenial
tags:
dynamic:
patch_set: true
- job:
name: armada-docker-build-gate-opensuse
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
vars:
publish: false
distro: opensuse_15
distro: ubuntu_focal
tags:
dynamic:
patch_set: true
- job:
name: armada-airskiff-deploy
nodeset: armada-single-node
nodeset: armada-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
@ -127,50 +119,33 @@
- ^releasenotes/.*$
- ^swagger/.*$
- job:
name: armada-docker-publish-ubuntu_bionic
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: ubuntu_bionic
tags:
dynamic:
branch: true
commit: true
static:
- latest
# - job:
# name: armada-docker-publish-ubuntu_bionic
# timeout: 1800
# run: tools/gate/playbooks/docker-image-build.yaml
# nodeset: armada-single-node
# secrets:
# - airship_armada_quay_creds
# vars:
# publish: true
# distro: ubuntu_bionic
# tags:
# dynamic:
# branch: true
# commit: true
# static:
# - latest
- job:
name: armada-docker-publish-ubuntu_xenial
name: armada-docker-publish-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
nodeset: armada-single-node-focal
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: ubuntu_xenial
tags:
dynamic:
branch: true
commit: true
static:
- latest
- job:
name: armada-docker-publish-opensuse
timeout: 1800
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: armada-single-node
secrets:
- airship_armada_quay_creds
vars:
publish: true
distro: opensuse_15
distro: ubuntu_focal
tags:
dynamic:
branch: true

4
Makefile
View File

@ -28,7 +28,7 @@ LABEL ?= org.airshipit.build=community
COMMIT ?= $(shell git rev-parse HEAD)
PYTHON = python3
CHARTS := $(filter-out deps, $(patsubst charts/%/.,%,$(wildcard charts/*/.)))
DISTRO ?= ubuntu_bionic
DISTRO ?= ubuntu_focal
IMAGE := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
UBUNTU_BASE_IMAGE ?=
@ -135,7 +135,7 @@ test-all: check-tox helm_lint
tox
test-unit: check-tox
tox -e py35
tox -e py38
test-coverage: check-tox
tox -e cover

10
armada/exceptions/base_exception.py
View File

@ -14,6 +14,7 @@
import json
import traceback
from contextlib import suppress
import falcon
from oslo_config import cfg
@ -251,10 +252,13 @@ class ArmadaBaseException(Exception):
def __init__(self, message=None, **kwargs):
self.message = message or self.message
try: # nosec
# replacing try-except-pass block with suppress
with suppress(Exception):
self.message = self.message % kwargs
except Exception:
pass
# try: # nosec
# self.message = self.message % kwargs
# except Exception:
# pass
super(ArmadaBaseException, self).__init__(self.message)

9
armada/tests/unit/utils/test_source.py
View File

@ -110,7 +110,7 @@ class GitTestCase(base.ArmadaTestCase):
source.download_tarball(url)
mock_temp.mkstemp.assert_called_once()
mock_requests.get.assert_called_once_with(url, verify=False)
mock_requests.get.assert_called_once_with(url, timeout=5, verify=False)
mock_open.assert_called_once_with('/tmp/armada', 'wb')
mock_open().write.assert_called_once_with(
mock_requests.get(url).content)
@ -121,7 +121,8 @@ class GitTestCase(base.ArmadaTestCase):
def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp):
mock_path.exists.return_value = True
mock_temp.mkdtemp.return_value = '/tmp/armada'
mock_opened_file = mock.Mock()
mock_opened_file = mock.MagicMock()
mock_opened_file.__iter__.return_value = ['file']
mock_tarfile.open.return_value = mock_opened_file
path = '/tmp/mariadb-0.1.0.tgz'
@ -130,7 +131,7 @@ class GitTestCase(base.ArmadaTestCase):
mock_path.exists.assert_called_once()
mock_temp.mkdtemp.assert_called_once()
mock_tarfile.open.assert_called_once_with(path)
mock_opened_file.extractall.assert_called_once_with('/tmp/armada')
mock_opened_file.extract.assert_called_once_with('file', '/tmp/armada')
@test_utils.attr(type=['negative'])
@mock.patch('armada.utils.source.os.path')
@ -144,7 +145,7 @@ class GitTestCase(base.ArmadaTestCase):
path)
mock_tarfile.open.assert_not_called()
mock_tarfile.extractall.assert_not_called()
mock_tarfile.extract.assert_not_called()
@testtools.skipUnless(
base.is_connected(), 'git clone requires network connectivity.')

6
armada/utils/source.py
View File

@ -134,7 +134,8 @@ def download_tarball(tarball_url, verify=False, proxy_server=None):
'ftp': proxy_server
}
tarball_filename = tempfile.mkstemp(prefix='armada')[1]
response = requests.get(tarball_url, verify=verify, **kwargs)
response = requests.get(
tarball_url, timeout=5, verify=verify, **kwargs)
with open(tarball_filename, 'wb') as f:
f.write(response.content)
@ -155,7 +156,8 @@ def extract_tarball(tarball_path):
try:
file = tarfile.open(tarball_path)
file.extractall(temp_dir)
for member in file:
file.extract(member, temp_dir)
except Exception:
raise source_exceptions.TarballExtractException(tarball_path)
return temp_dir

2
armada/utils/validate.py
View File

@ -177,6 +177,6 @@ def validate_armada_documents(documents):
def validate_manifest_url(value):
try:
return (requests.get(value).status_code == 200)
return (requests.get(value, timeout=5).status_code == 200)
except requests.exceptions.RequestException:
return False

85
images/armada/Dockerfile.opensuse_15
View File

@ -1,85 +0,0 @@
ARG FROM=opensuse/leap:15.1
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
org.opencontainers.image.url='https://airshipit.org' \
org.opencontainers.image.documentation='https://docs.airshipit.org/armada' \
org.opencontainers.image.source='https://opendev.org/airship/armada' \
org.opencontainers.image.vendor='The Airship Authors' \
org.opencontainers.image.licenses='Apache-2.0'
ENV LANG=C.UTF-8
ENV LC_ALL=C.UTF-8
EXPOSE 8000
ENTRYPOINT ["./entrypoint.sh"]
CMD ["server"]
RUN mkdir -p /armada && \
zypper refresh && \
zypper up -y && \
zypper --non-interactive install \
ca-certificates \
curl \
tar \
gzip \
netcfg \
python3 \
python3-devel \
python3-setuptools \
python3-pip \
automake \
gcc \
git \
libopenssl-devel \
libtool \
make
RUN pip3 install --upgrade pip && \
zypper clean -a && \
rm -rf \
/tmp/* \
/var/tmp/* \
/usr/share/man \
/usr/share/doc \
/usr/share/doc-base
ARG HELM_ARTIFACT_URL
RUN \
curl -fSSL -O ${HELM_ARTIFACT_URL} && \
tar -xvf $(basename ${HELM_ARTIFACT_URL}) && \
mv linux-amd64/helm /usr/local/bin
ENV LD_LIBRARY_PATH=/usr/local/lib
ARG LIBYAML_VERSION=0.2.5
RUN set -ex \
&& git clone https://github.com/yaml/libyaml.git \
&& cd libyaml \
&& git checkout $LIBYAML_VERSION \
&& ./bootstrap \
&& ./configure \
&& make \
&& make install \
&& cd .. \
&& rm -fr libyaml
WORKDIR /armada
COPY requirements.txt /tmp/
RUN \
pip3 install -r /tmp/requirements.txt && \
useradd -u 1000 -g users -d /armada armada && \
rm -rf /tmp/requirements.txt
COPY . /armada
RUN \
mv etc/armada /etc/ && \
cd /armada && \
chown -R armada:users /armada && \
python3 setup.py install
USER armada

4
images/armada/Dockerfile.ubuntu_xenial → images/armada/Dockerfile.ubuntu_focal
View File

@ -1,4 +1,4 @@
ARG FROM=ubuntu:16.04
ARG FROM=ubuntu:20.04
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@ -70,7 +70,7 @@ RUN set -ex \
&& make install \
&& cd .. \
&& rm -fr libyaml \
&& python3 -m pip install -U 'pip<21.0' \
&& python3 -m pip install -U pip \
&& pip3 install -r requirements.txt --no-cache-dir \
&& curl -fSSL -O ${HELM_ARTIFACT_URL} \
&& tar -xvf $(basename ${HELM_ARTIFACT_URL}) \

6
requirements.txt
View File

@ -1,11 +1,11 @@
amqp<2.7,>=2.6.0
amqp==5.0.8
deepdiff>=3.3.0
gitpython
jsonschema>=3.0.1,<4
keystoneauth1>=3.18.0
keystonemiddleware>=5.3.0
kombu<4.7,>=4.6.10
kubernetes~=25.3.0; python_version >= '3.6'
kombu==5.1.0
kubernetes~=26.1.0; python_version >= '3.6'
Paste>=2.0.3
PasteDeploy>=1.5.2
pylibyaml~=0.1

11
setup.cfg
View File

@ -1,10 +1,11 @@
[metadata]
name = Armada
version = 1.0
summary = Tool for managing multiple Helm charts with dependencies by centralizing all configurations in a single Armada YAML and providing life-cycle hooks for all Helm releases.
description-file = README.rst
description_file = README.rst
author = The Airship Authors
author-email = airship-discuss@lists.airshipit.org
home-page = https://airshipit.org
author_email = airship-discuss@lists.airshipit.org
home_page = https://airshipit.org
classifier =
Intended Audience :: Information Technology
@ -13,8 +14,8 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
[files]
packages =

11
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -13,6 +13,17 @@
# limitations under the License.
- hosts: primary
vars:
env:
HTTP_PROXY: ""
HTTPS_PROXY: ""
NO_PROXY: ""
DISTRO: "{{ distro }}"
become: true
roles:
- clear-firewall
tasks:
# Stop systemd-resolved service before starting docker.
- name: stop systemd-resolved service

4
tools/gate/playbooks/docker-image-build.yaml
View File

@ -72,7 +72,11 @@
name: "{{ item }}"
allow_unauthenticated: True
with_items:
- apparmor
- docker-ce
- docker-ce-cli
- docker-buildx-plugin
- containerd.io
- python3-pip
- python3-setuptools
- pip:

2
tools/helm_tk.sh
View File

@ -17,7 +17,7 @@
set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"}
TMP_DIR=$(mktemp -d)

10
tox.ini
View File

@ -7,7 +7,13 @@ envlist = py38, pep8, cover, bandit
deps=
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
passenv=HTTP_PROXY,HTTPS_PROXY,http_proxy,https_proxy,NO_PROXY,no_proxy
passenv=
HTTP_PROXY
HTTPS_PROXY
http_proxy
https_proxy
NO_PROXY
no_proxy
setenv=
VIRTUAL_ENV={envdir}
usedevelop = True
@ -101,7 +107,7 @@ show-source = true
# [H904] Delay string interpolations at logging calls.
enable-extensions = H106,H201,H904
# [W503] line break before binary operator
ignore = W503
ignore = W503,B202,B113
exclude = .git,.tox,dist,*lib/python*,*egg,build,releasenotes,doc/*,venv
max-complexity = 24
application-import-names = armada