From 5ead2930ae0ab5e00507b204b1efcecde9f284b3 Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Sat, 3 Jun 2023 21:33:12 +0000 Subject: [PATCH] Restore bionic images This PS restores building ans testing bionic images Change-Id: Id88e1c096a8934e5974b9404cc2b0bfd748fc25f --- .gitignore | 1 + Makefile | 2 +- charts/calicoctl-utility/values.yaml | 2 +- charts/ceph-utility/values.yaml | 2 +- charts/compute-utility/values.yaml | 2 +- charts/etcdctl-utility/values.yaml | 2 +- charts/mysqlclient-utility/values.yaml | 4 +- charts/openstack-utility/values.yaml | 2 +- charts/postgresql-utility/values.yaml | 2 +- .../Dockerfile.ubuntu_bionic | 53 +++++++ ...Dockerfile_calicoq_calicoctl.ubuntu_bionic | 61 ++++++++ images/ceph-utility/Dockerfile.ubuntu_bionic | 66 +++++++++ .../compute-utility/Dockerfile.ubuntu_bionic | 55 +++++++ .../etcdctl-utility/Dockerfile.ubuntu_bionic | 56 +++++++ .../Dockerfile.ubuntu_bionic | 60 ++++++++ .../Dockerfile.ubuntu_bionic | 39 +++++ .../Dockerfile.ubuntu_bionic | 53 +++++++ .../unit/services/test_utility_dataloader.py | 2 + .../test_mysqlclient_utility_container.py | 5 +- .../test_postgresql_utility_container.py | 64 ++++---- tools/deployment/004-make-images.sh | 17 +++ tools/deployment/005-calicoctl-utility.sh | 3 +- tools/deployment/010-ceph-utility.sh | 3 +- tools/deployment/020-compute-utility.sh | 3 +- tools/deployment/030-etcdctl-utility.sh | 3 +- tools/deployment/040-mysqlclient-utility.sh | 44 +++++- tools/deployment/050-openstack-utility.sh | 3 +- tools/deployment/060-postgresql-utility.sh | 15 +- tools/gate/deploy.sh | 3 +- .../airship-porthole-gate-runner.yaml | 20 +-- .../gate/playbooks/files/docker-systemd.conf | 8 - tools/gate/playbooks/install-deps.yaml | 4 + tools/gate/playbooks/make-feature-tests.yaml | 13 +- tools/gate/playbooks/make-unit-tests.yaml | 13 +- .../playbooks/osh-infra-upgrade-host.yaml | 55 ------- .../roles/osh-run-script/defaults/main.yaml | 2 +- .../roles/osh-run-script/tasks/main.yaml | 1 + .../roles/setup-firewall/tasks/main.yaml | 29 ---- tools/run_avt.sh | 20 +-- tox.ini | 5 +- zuul.d/base.yaml | 138 ++++++++++++++---- zuul.d/calicoctl-utility.yaml | 39 ++++- zuul.d/ceph-utility.yaml | 38 ++++- zuul.d/compute-utility.yaml | 38 ++++- zuul.d/etcdctl-utility.yaml | 38 ++++- zuul.d/mysqlclient-utility.yaml | 38 ++++- zuul.d/openstack-utility.yaml | 38 ++++- zuul.d/postgresql-utility.yaml | 108 +++++++++----- 48 files changed, 999 insertions(+), 273 deletions(-) create mode 100644 images/calicoctl-utility/Dockerfile.ubuntu_bionic create mode 100644 images/calicoctl-utility/Dockerfile_calicoq_calicoctl.ubuntu_bionic create mode 100755 images/ceph-utility/Dockerfile.ubuntu_bionic create mode 100755 images/compute-utility/Dockerfile.ubuntu_bionic create mode 100644 images/etcdctl-utility/Dockerfile.ubuntu_bionic create mode 100644 images/mysqlclient-utility/Dockerfile.ubuntu_bionic create mode 100755 images/openstack-utility/Dockerfile.ubuntu_bionic create mode 100644 images/postgresql-utility/Dockerfile.ubuntu_bionic create mode 100755 tools/deployment/004-make-images.sh delete mode 100644 tools/gate/playbooks/files/docker-systemd.conf delete mode 100644 tools/gate/playbooks/osh-infra-upgrade-host.yaml delete mode 100644 tools/gate/playbooks/roles/setup-firewall/tasks/main.yaml diff --git a/.gitignore b/.gitignore index f73de10e..2d49023c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ cover/ nosetests.xml .testrepository .venv +.stestr # Translations *.mo diff --git a/Makefile b/Makefile index 5902a0e5..fa38a5af 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ unit_tests: feature_tests: @echo "Run Feature Validation Testing" ./tools/run_avt.sh feature_tests - tox -e docs + format: @echo "Not implemented." >&2; exit 2 diff --git a/charts/calicoctl-utility/values.yaml b/charts/calicoctl-utility/values.yaml index c12aec5e..ec7c6795 100644 --- a/charts/calicoctl-utility/values.yaml +++ b/charts/calicoctl-utility/values.yaml @@ -17,7 +17,7 @@ images: tags: - calicoctl_utility: 'quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_bionic' + calicoctl_utility: 'quay.io/airshipit/porthole-calicoctl-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 pull_policy: IfNotPresent local_registry: diff --git a/charts/ceph-utility/values.yaml b/charts/ceph-utility/values.yaml index aae7e02b..9c6ea71b 100644 --- a/charts/ceph-utility/values.yaml +++ b/charts/ceph-utility/values.yaml @@ -20,7 +20,7 @@ release_group: null images: pull_policy: IfNotPresent tags: - ceph_utility: 'quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_bionic' + ceph_utility: 'quay.io/airshipit/porthole-ceph-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 local_registry: active: false diff --git a/charts/compute-utility/values.yaml b/charts/compute-utility/values.yaml index 95cb4203..07ca3a0f 100644 --- a/charts/compute-utility/values.yaml +++ b/charts/compute-utility/values.yaml @@ -21,7 +21,7 @@ release_uuid: null images: pull_policy: IfNotPresent tags: - compute_utility: 'quay.io/airshipit/porthole-compute-utility:latest-ubuntu_bionic' + compute_utility: 'quay.io/airshipit/porthole-compute-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 local_registry: active: false diff --git a/charts/etcdctl-utility/values.yaml b/charts/etcdctl-utility/values.yaml index 0441a08e..98a4e9fd 100644 --- a/charts/etcdctl-utility/values.yaml +++ b/charts/etcdctl-utility/values.yaml @@ -20,7 +20,7 @@ release_group: null images: pull_policy: IfNotPresent tags: - etcdctl_utility: 'quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_bionic' + etcdctl_utility: 'quay.io/airshipit/porthole-etcdctl-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 pull_policy: "IfNotPresent" local_registry: diff --git a/charts/mysqlclient-utility/values.yaml b/charts/mysqlclient-utility/values.yaml index b8839f86..18e4169c 100644 --- a/charts/mysqlclient-utility/values.yaml +++ b/charts/mysqlclient-utility/values.yaml @@ -19,8 +19,8 @@ release_group: null images: tags: - mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_bionic - mysqlclient_utility: 'quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_bionic' + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + mysqlclient_utility: 'quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 pull_policy: IfNotPresent local_registry: diff --git a/charts/openstack-utility/values.yaml b/charts/openstack-utility/values.yaml index 61c2e093..8b24c3e5 100644 --- a/charts/openstack-utility/values.yaml +++ b/charts/openstack-utility/values.yaml @@ -9,7 +9,7 @@ release_group: null images: pull_policy: IfNotPresent tags: - openstack_utility: 'quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_bionic' + openstack_utility: 'quay.io/airshipit/porthole-openstack-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 local_registry: active: false diff --git a/charts/postgresql-utility/values.yaml b/charts/postgresql-utility/values.yaml index bc4c3754..788d3477 100644 --- a/charts/postgresql-utility/values.yaml +++ b/charts/postgresql-utility/values.yaml @@ -20,7 +20,7 @@ release_group: null images: pull_policy: IfNotPresent tags: - postgresql_utility: 'quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic' + postgresql_utility: 'quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_focal' image_repo_sync: docker.io/docker:18.09.02 local_registry: active: false diff --git a/images/calicoctl-utility/Dockerfile.ubuntu_bionic b/images/calicoctl-utility/Dockerfile.ubuntu_bionic new file mode 100644 index 00000000..7a9210e9 --- /dev/null +++ b/images/calicoctl-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,53 @@ +ARG FROM=docker.io/ubuntu:bionic +ARG DOCKER_REGISTRY=quay.io +ARG CALICOCTL_IMAGE_PREFIX=calico/ctl +ARG CALICOCTL_VERSION=v3.23.1 +ARG CALICOCTL_IMAGE="${DOCKER_REGISTRY}/${CALICOCTL_IMAGE_PREFIX}:${CALICOCTL_VERSION}" + +FROM ${CALICOCTL_IMAGE} as stage_calicoctl + +FROM ${FROM} + +ARG CALICOCTL_VERSION + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' \ + org.projectcalico.calicoctl.version="${CALICOCTL_VERSION}" + +RUN set -xe \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + locales \ + moreutils \ + sudo \ + socat \ + python3.6 \ + python3-pip \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +COPY --from=stage_calicoctl /calicoctl /usr/local/bin/calicoctl + +RUN set -xe \ + && chmod 0754 /usr/local/bin/calicoctl \ + && chmod 0754 /usr/bin/socat + +WORKDIR /tmp + +CMD ["/bin/bash"] diff --git a/images/calicoctl-utility/Dockerfile_calicoq_calicoctl.ubuntu_bionic b/images/calicoctl-utility/Dockerfile_calicoq_calicoctl.ubuntu_bionic new file mode 100644 index 00000000..62f3517b --- /dev/null +++ b/images/calicoctl-utility/Dockerfile_calicoq_calicoctl.ubuntu_bionic @@ -0,0 +1,61 @@ +ARG FROM=docker.io/ubuntu:bionic +ARG DOCKER_REGISTRY=quay.io +ARG CALICOCTL_IMAGE_PREFIX=tigera/calicoctl +ARG CALICOCTL_VERSION=v3.23.1 +ARG CALICOCTL_IMAGE="${DOCKER_REGISTRY}/${CALICOCTL_IMAGE_PREFIX}:${CALICOCTL_VERSION}" +ARG CALICOQ_IMAGE_PREFIX=tigera/calicoq +ARG CALICOQ_VERSION=v3.23.1 +ARG CALICOQ_IMAGE="${DOCKER_REGISTRY}/${CALICOQ_IMAGE_PREFIX}:${CALICOQ_VERSION}" + +FROM ${CALICOCTL_IMAGE} as stage_calicoctl +FROM ${CALICOQ_IMAGE} as stage_calicoq + +FROM ${FROM} + +ARG CALICOCTL_VERSION +ARG CALICOQ_VERSION + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' \ + io.tigera.calicoctl.version="${CALICOCTL_VERSION}" \ + io.tigera.calicoq.version="${CALICOQ_VERSION}" + +RUN set -xe \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + locales \ + moreutils \ + sudo \ + socat \ + python3.6 \ + python3-pip \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +COPY --from=stage_calicoctl /calicoctl /usr/local/bin/calicoctl +COPY --from=stage_calicoq /calicoq /usr/local/bin/calicoq + +RUN set -xe \ + && chmod 0754 /usr/local/bin/calicoctl \ + && chmod 0754 /usr/local/bin/calicoq \ + && chmod 0754 /usr/bin/socat + +WORKDIR /tmp + +CMD ["/bin/bash"] diff --git a/images/ceph-utility/Dockerfile.ubuntu_bionic b/images/ceph-utility/Dockerfile.ubuntu_bionic new file mode 100755 index 00000000..a52800df --- /dev/null +++ b/images/ceph-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,66 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +# Pacific 16.2.10 +ARG CEPH_RELEASE=pacific +ARG CEPH_RELEASE_TAG=16.2.10-1bionic +ARG KUBE_VERSION=1.24.6 + +ARG CEPH_REPO=https://mirror.mirantis.com/acicd/ceph-pacific/ +ARG CEPH_KEY=https://mirror.mirantis.com/acicd/ceph-pacific/release.asc + +ADD ${CEPH_KEY} /etc/apt/ceph-${CEPH_RELEASE}.key +RUN set -xe \ + && export DEBIAN_FRONTEND=noninteractive \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update && apt-get dist-upgrade -y \ + && apt-get install -y wget curl apt-transport-https ca-certificates gnupg\ + && apt-key add /etc/apt/ceph-${CEPH_RELEASE}.key \ + && rm -f /etc/apt/ceph-${CEPH_RELEASE}.key \ + && echo "deb ${CEPH_REPO} bionic main" | tee /etc/apt/sources.list.d/ceph.list \ + && apt-get update \ + && apt-get install -y \ + bash \ + moreutils \ + vim \ + sudo \ + screen \ + ceph=${CEPH_RELEASE_TAG} \ + ceph-common=${CEPH_RELEASE_TAG} \ + python3-rbd \ + radosgw=${CEPH_RELEASE_TAG} \ + hexedit \ + jq \ + s3cmd \ + rsyslog \ + rsync \ + xz-utils \ + iperf \ + python3.6 \ + python3-pip \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + && apt-get remove --purge -y wget apt-transport-https \ + && apt-get autoremove -y \ + && apt-get clean \ + && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ + | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ + && rm -rf \ + /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/images/compute-utility/Dockerfile.ubuntu_bionic b/images/compute-utility/Dockerfile.ubuntu_bionic new file mode 100755 index 00000000..3419d216 --- /dev/null +++ b/images/compute-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,55 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG KUBE_VERSION=1.24.6 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -xe \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y apt-transport-https \ + bash \ + ca-certificates \ + openvswitch-switch \ + curl \ + gnupg \ + hexedit \ + iperf \ + jq \ + moreutils \ + radosgw \ + rsyslog \ + s3cmd \ + rsync \ + sudo \ + wget \ + xz-utils \ + python3.6 \ + python3-pip \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + && apt-get remove --purge -y wget apt-transport-https \ + && apt-get autoremove -y \ + && apt-get clean \ + && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ + | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ + && rm -rf \ + /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/images/etcdctl-utility/Dockerfile.ubuntu_bionic b/images/etcdctl-utility/Dockerfile.ubuntu_bionic new file mode 100644 index 00000000..845610aa --- /dev/null +++ b/images/etcdctl-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,56 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG ETCDCTL_VERSION=v3.5.3 +ARG KUBE_VERSION=1.24.6 + +ENV GOOGLE_URL=https://storage.googleapis.com/etcd +ENV GITHUB_URL=https://github.com/etcd-io/etcd/releases/download +ENV DOWNLOAD_URL=${GITHUB_URL} + +RUN set -xe \ + && export DEBIAN_FRONTEND=noninteractive \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y \ + wget curl \ + locales \ + apt-transport-https \ + ca-certificates \ + gnupg \ + bash \ + moreutils \ + sudo \ + rsyslog \ + python3.6 \ + python3-pip \ + jq \ + rclone \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + python-openstackclient==5.4.0 \ + python-swiftclient==3.10.1 \ + && curl --silent -L "${DOWNLOAD_URL}/${ETCDCTL_VERSION}/etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz" \ + | tar -zC /usr/local/bin --strip-components=1 --wildcards -x "*/etcdctl" \ + && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ + | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ + && apt-get clean -y \ + && rm -rf \ + /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/images/mysqlclient-utility/Dockerfile.ubuntu_bionic b/images/mysqlclient-utility/Dockerfile.ubuntu_bionic new file mode 100644 index 00000000..fad571aa --- /dev/null +++ b/images/mysqlclient-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,60 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG KUBE_VERSION=1.24.6 + +RUN set -xe \ + && export DEBIAN_FRONTEND=noninteractive \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y wget curl \ + apt-transport-https ca-certificates gnupg \ + && curl -o /etc/apt/trusted.gpg.d/mariadb_release_signing_key.asc 'https://mariadb.org/mariadb_release_signing_key.asc' \ + && echo 'deb https://mirrors.gigenet.com/mariadb/repo/10.6/ubuntu bionic main' >>/etc/apt/sources.list \ + && apt-get update \ + && apt-get install -y \ + bash \ + sudo \ + rsyslog \ + pwgen \ + curl \ + libdbi-perl \ + libdbd-mysql-perl \ + moreutils \ + mysql-client \ + python3.6 \ + python3-pip \ + jq \ + rclone \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + python-openstackclient==5.4.0 \ + python-swiftclient==3.10.1 \ + && sed -i 's/$PrivDropToUser syslog/$PrivDropToUser nobody/' /etc/rsyslog.conf \ + && apt-get clean -y \ + && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ + | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ + && curl --silent -L https://percona.com/get/percona-toolkit.tar.gz \ + | tar -zC /usr/local/bin/ --strip-components=2 --wildcards -x "*/*/pt-show-grants" \ + && rm -rf \ + /tmp/* \ + /var/cache/debconf/* \ + /var/lib/apt/lists/* \ + /var/tmp/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/images/openstack-utility/Dockerfile.ubuntu_bionic b/images/openstack-utility/Dockerfile.ubuntu_bionic new file mode 100755 index 00000000..5fd2e15c --- /dev/null +++ b/images/openstack-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,39 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG DEBIAN_FRONTEND=noninteractive + +RUN set -xe \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y \ + bash \ + sudo \ + rsyslog \ + python3.6 \ + python3-pip \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + python-openstackclient==5.4.0 \ + python-glanceclient==3.2.2 \ + python-novaclient==17.2.1 \ + python-neutronclient==7.2.1 \ + python-cinderclient==7.2.2 \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i "/rootwrap_logger.setLevel/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" /usr/local/lib/python3.6/dist-packages/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/images/postgresql-utility/Dockerfile.ubuntu_bionic b/images/postgresql-utility/Dockerfile.ubuntu_bionic new file mode 100644 index 00000000..a221fcbe --- /dev/null +++ b/images/postgresql-utility/Dockerfile.ubuntu_bionic @@ -0,0 +1,53 @@ +ARG FROM=docker.io/ubuntu:bionic +FROM ${FROM} + +LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \ + org.opencontainers.image.url='https://airshipit.org' \ + org.opencontainers.image.documentation='https://opendev.org/airship/porthole' \ + org.opencontainers.image.source='https://opendev.org/airship/porthole' \ + org.opencontainers.image.vendor='The Airship Authors' \ + org.opencontainers.image.licenses='Apache-2.0' + +ARG KUBE_VERSION=1.24.6 + +RUN set -xe \ + && export DEBIAN_FRONTEND=noninteractive \ + && sed -i '/nobody/d' /etc/passwd \ + && echo "nobody:x:65534:65534:nobody:/nonexistent:/bin/bash" >> /etc/passwd \ + && apt-get update \ + && apt-get install -y wget curl \ + apt-transport-https ca-certificates gnupg \ + && echo "deb http://apt.postgresql.org/pub/repos/apt bionic-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \ + && apt-get update \ + && apt-get install -y \ + bash \ + moreutils \ + rsyslog \ + screen \ + sudo \ + postgresql-client \ + postgresql-common \ + python3.6 \ + python3-pip \ + jq \ + rclone \ + && pip3 install --upgrade pip \ + && pip3 install \ + oslo.rootwrap==6.2.0 \ + python-openstackclient==5.4.0 \ + python-swiftclient==3.10.1 \ + && apt-get clean -y \ + && curl --silent -L https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz \ + | tar -zC /usr/bin --strip-components=3 --wildcards -x "*/*/*/kubectl" \ + && rm -rf \ + /var/cache/debconf/* \ + /var/lib/apt/lists/* + +RUN PYTHON_LOCATION=$(pip3 show oslo.rootwrap|grep Location|awk '{print $2}') \ + && sed -i "/rootwrap_logger.setLevel/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/handler.setFormatter/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/os.path.basename/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py \ + && sed -i "/rootwrap_logger.addHandler/s/.*/#&/" $PYTHON_LOCATION/oslo_rootwrap/wrapper.py + +CMD ["/bin/bash"] diff --git a/kube_utility_container/tests/unit/services/test_utility_dataloader.py b/kube_utility_container/tests/unit/services/test_utility_dataloader.py index 1834d731..927facef 100644 --- a/kube_utility_container/tests/unit/services/test_utility_dataloader.py +++ b/kube_utility_container/tests/unit/services/test_utility_dataloader.py @@ -26,8 +26,10 @@ class TestDeploymentNameMapping(unittest.TestCase): def setUp(self) -> None: self.mapping = DeploymentMapping(self) + super().setUp() def tearDown(self) -> None: + super().tearDown() pass def test_deployment_name_is_consistent_with_name_mapping(self): diff --git a/kube_utility_container/tests/utility/mysqlclient/test_mysqlclient_utility_container.py b/kube_utility_container/tests/utility/mysqlclient/test_mysqlclient_utility_container.py index 28b54575..844d4ac7 100644 --- a/kube_utility_container/tests/utility/mysqlclient/test_mysqlclient_utility_container.py +++ b/kube_utility_container/tests/utility/mysqlclient/test_mysqlclient_utility_container.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import time from kube_utility_container.tests.utility.base import TestBase @@ -70,10 +71,12 @@ class TestMysqlclientUtilityContainer(TestBase): date_2 = (self.client.exec_cmd(self.deployment_name, ['date', '+%b %d %H'])).replace( '\n', '') - exec_cmd = ['utilscli', 'mysql', 'version'] + exec_cmd = ['utilscli', 'dbutils', 'show_databases', 'openstack'] self.client.exec_cmd(self.deployment_name, exec_cmd) + time.sleep(10) pod_logs = (self.client._get_pod_logs(self.deployment_name)). \ replace('\n', '') + print(pod_logs) if date_1 in pod_logs: latest_pod_logs = (pod_logs.split(date_1))[1:] else: diff --git a/kube_utility_container/tests/utility/postgresql/test_postgresql_utility_container.py b/kube_utility_container/tests/utility/postgresql/test_postgresql_utility_container.py index 0fe3885a..ae637797 100644 --- a/kube_utility_container/tests/utility/postgresql/test_postgresql_utility_container.py +++ b/kube_utility_container/tests/utility/postgresql/test_postgresql_utility_container.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kube_utility_container.tests.utility.base import TestBase +import time -import warnings +from kube_utility_container.tests.utility.base import TestBase class TestPostgresqlUtilityContainer(TestBase): @@ -23,6 +23,15 @@ class TestPostgresqlUtilityContainer(TestBase): cls.deployment_name = cls._get_deployment_name("postgresql-utility") super(TestPostgresqlUtilityContainer, cls).setUpClass() + def test_verify_postgresql_client_psql_is_present(self): + """To verify psql-client is present""" + exec_cmd = ['utilscli', 'psql', '-V'] + expected = 'psql' + result_set = self.client.exec_cmd(self.deployment_name, exec_cmd) + self.assertIn( + expected, result_set, 'Unexpected value for command: {}, ' + 'Command Output: {}'.format(exec_cmd, result_set)) + def test_verify_readonly_rootfs(self): """To verify postgresql-utility readonly rootfs configuration""" failures = [] @@ -38,36 +47,6 @@ class TestPostgresqlUtilityContainer(TestBase): f" in pod {postgresql_utility_pod.metadata.name}") self.assertEqual(0, len(failures), failures) - def test_verify_postgresql_utility_pod_logs(self): - """To verify postgresql-utility pod logs""" - warnings.filterwarnings( - action="ignore", message="unclosed", category=ResourceWarning) - date_1 = (self.client.exec_cmd(self.deployment_name, - ['date', '+%Y-%m-%d %H'])).replace( - '\n', '') - date_2 = (self.client.exec_cmd(self.deployment_name, - ['date', '+%b %d %H'])).replace( - '\n', '') - exec_cmd = ['utilscli', 'psql', 'version'] - self.client.exec_cmd(self.deployment_name, exec_cmd) - pod_logs = (self.client._get_pod_logs(self.deployment_name)). \ - replace('\n', '') - if date_1 in pod_logs: - latest_pod_logs = (pod_logs.split(date_1))[1:] - else: - latest_pod_logs = (pod_logs.split(date_2))[1:] - self.assertNotEqual(0, len(latest_pod_logs), - "Not able to get the latest logs") - - def test_verify_postgresql_client_psql_is_present(self): - """To verify psql-client is present""" - exec_cmd = ['utilscli', 'psql', '-V'] - expected = 'psql' - result_set = self.client.exec_cmd(self.deployment_name, exec_cmd) - self.assertIn( - expected, result_set, 'Unexpected value for command: {}, ' - 'Command Output: {}'.format(exec_cmd, result_set)) - def test_verify_apparmor(self): """To verify postgresql-utility Apparmor""" failures = [] @@ -84,3 +63,24 @@ class TestPostgresqlUtilityContainer(TestBase): f"{postgresql_utility_pod.metadata.name} " f"is not having expected apparmor profile set") self.assertEqual(0, len(failures), failures) + + def test_verify_postgresql_utility_pod_logs(self): + """To verify postgresql-utility pod logs""" + date_1 = (self.client.exec_cmd(self.deployment_name, + ['date', '+%Y-%m-%d %H'])).replace( + '\n', '') + date_2 = (self.client.exec_cmd(self.deployment_name, + ['date', '+%b %d %H'])).replace( + '\n', '') + exec_cmd = ['utilscli', 'dbutils', 'show_databases'] + self.client.exec_cmd(self.deployment_name, exec_cmd) + time.sleep(10) + pod_logs = (self.client._get_pod_logs(self.deployment_name)). \ + replace('\n', '') + print(pod_logs) + if date_1 in pod_logs: + latest_pod_logs = (pod_logs.split(date_1))[1:] + else: + latest_pod_logs = (pod_logs.split(date_2))[1:] + self.assertNotEqual(0, len(latest_pod_logs), + "Not able to get the latest logs") diff --git a/tools/deployment/004-make-images.sh b/tools/deployment/004-make-images.sh new file mode 100755 index 00000000..7a5faf3c --- /dev/null +++ b/tools/deployment/004-make-images.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -x + +env +make images \ No newline at end of file diff --git a/tools/deployment/005-calicoctl-utility.sh b/tools/deployment/005-calicoctl-utility.sh index 43b27173..eb1caa38 100755 --- a/tools/deployment/005-calicoctl-utility.sh +++ b/tools/deployment/005-calicoctl-utility.sh @@ -13,7 +13,8 @@ set -xe namespace=utility -helm upgrade --install calicoctl-utility ./artifacts/calicoctl-utility.tgz --namespace=$namespace +helm upgrade --install calicoctl-utility ./artifacts/calicoctl-utility.tgz --namespace=$namespace \ + --set "images.tags.calicoctl_utility=quay.io/airshipit/porthole-calicoctl-utility:latest-${DISTRO}" # Wait for Deployment : "${OSH_INFRA_PATH:="../openstack-helm-infra"}" diff --git a/tools/deployment/010-ceph-utility.sh b/tools/deployment/010-ceph-utility.sh index 71a8e4d4..dbc09a16 100755 --- a/tools/deployment/010-ceph-utility.sh +++ b/tools/deployment/010-ceph-utility.sh @@ -52,7 +52,8 @@ helm upgrade --install ceph-utility-config ./ceph-provisioners \ # Deploy Ceph-Utility cd ${CURRENT_DIR} -helm upgrade --install ceph-utility ./artifacts/ceph-utility.tgz --namespace=$namespace +helm upgrade --install ceph-utility ./artifacts/ceph-utility.tgz --namespace=$namespace \ + --set "images.tags.ceph_utility=quay.io/airshipit/porthole-ceph-utility:latest-${DISTRO}" # Wait for Deployment : "${OSH_INFRA_PATH:="../openstack-helm-infra"}" diff --git a/tools/deployment/020-compute-utility.sh b/tools/deployment/020-compute-utility.sh index 4e0d5bfb..9dacc46e 100755 --- a/tools/deployment/020-compute-utility.sh +++ b/tools/deployment/020-compute-utility.sh @@ -14,7 +14,8 @@ set -xe namespace="utility" -helm upgrade --install compute-utility ./artifacts/compute-utility.tgz --namespace=$namespace +helm upgrade --install compute-utility ./artifacts/compute-utility.tgz --namespace=$namespace \ + --set "images.tags.compute_utility=quay.io/airshipit/porthole-compute-utility:latest-${DISTRO}" # Wait for Deployment : "${OSH_INFRA_PATH:="../openstack-helm-infra"}" diff --git a/tools/deployment/030-etcdctl-utility.sh b/tools/deployment/030-etcdctl-utility.sh index bb84d633..f59e941d 100755 --- a/tools/deployment/030-etcdctl-utility.sh +++ b/tools/deployment/030-etcdctl-utility.sh @@ -13,7 +13,8 @@ set -xe namespace="utility" -helm upgrade --install etcdctl-utility ./artifacts/etcdctl-utility.tgz --namespace=$namespace +helm upgrade --install etcdctl-utility ./artifacts/etcdctl-utility.tgz --namespace=$namespace \ + --set "images.tags.etcdctl_utility=quay.io/airshipit/porthole-etcdctl-utility:latest-${DISTRO}" # Wait for Deployment : "${OSH_INFRA_PATH:="../openstack-helm-infra"}" diff --git a/tools/deployment/040-mysqlclient-utility.sh b/tools/deployment/040-mysqlclient-utility.sh index 0210b441..7001f278 100755 --- a/tools/deployment/040-mysqlclient-utility.sh +++ b/tools/deployment/040-mysqlclient-utility.sh @@ -12,8 +12,50 @@ # under the License. set -xe + +CURRENT_DIR="$(pwd)" + : "${OSH_PATH:="../openstack-helm"}" + +# Deploy mariadb server +cd "${OSH_PATH}" +tee /tmp/mariadb-server-config.yaml <