debian: use debian snapshot repos in docker apps

- build-wheels, build-docker-images: add command line options to enable the use of docker filesystem cache when building - use snapshot repo URLs defined by tools in build pods' environment to generate apt sources list files. These are defined in stx.conf. TESTS =============================== All tests were performed on Debian: - Build wheels - Build base image - Build a samnple docker app that derives from the base image Story: 2009897 Task: 45185 Depends-On: https://review.opendev.org/c/starlingx/tools/+/839396 Change-Id: I58a1ee002bb8161c492b3bcf8cd4bbcb6b4fcae4 Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
2022-04-22 18:16:00 -04:00 · 2022-04-22 18:16:00 -04:00 · 30d4eb3566
commit 30d4eb3566
parent 963bc08cc6
12 changed files with 158 additions and 39 deletions
--- a/build-tools/build-docker-images/build-stx-base.sh
+++ b/build-tools/build-docker-images/build-stx-base.sh
@ -238,7 +238,8 @@ if [ ${VALID_OS} -ne 0 ]; then
    exit 1
 fi

-SRC_DOCKERFILE=${MY_SCRIPT_DIR}/stx-${OS}/Dockerfile.${BUILD_STREAM}
+SRC_DOCKER_DIR="${MY_SCRIPT_DIR}/stx-${OS}"
+SRC_DOCKERFILE="${SRC_DOCKER_DIR}"/Dockerfile.${BUILD_STREAM}
 if [[ -z "$OS_VERSION" ]]; then
    OS_VERSION=$(
        sed -n -r 's/^\s*ARG\s+RELEASE\s*=\s*([^ \t#]+).*/\1/ip' $SRC_DOCKERFILE | head -n 1
@ -275,14 +276,8 @@ if [ ${#REPO_LIST[@]} -eq 0 ]; then
        if [[ "$OS" == "centos" ]] ; then
            REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
            REPO_LIST+=("stx-distro,http://${HOST}:8089${MY_REPO}/cgcs-${OS}-repo/Binary")
-        else
-            if [[ -z "$REPOMGR_DEPLOY_URL" ]] ; then
-                echo "Required env variable REPOMGR_DEPLOY_URL is not defined!" >&2
-                exit 1
-            fi
-            REPO_LIST+=("deb [trusted=yes] $REPOMGR_DEPLOY_URL/deb-local-binary bullseye main")
-            REPO_LIST+=("deb [trusted=yes] $REPOMGR_DEPLOY_URL/deb-local-build bullseye main")
        fi
+        # debian is handled down below
    elif [ "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then
        echo "Either --local or --repo must be specified" >&2
        exit 1
@ -336,11 +331,53 @@ EOF
        REPO_OPTS="${REPO_OPTS} --enablerepo=${repo_name}"
    done
 else
-    STX_APT_SOURCES_FILE=${BUILDDIR}/stx.apt.sources.list
-    rm -f "$STX_APT_SOURCES_FILE"
-    for repo in "${REPO_LIST[@]}" ; do
-        echo "$repo" >>"$STX_APT_SOURCES_FILE"
+
+    # These env vars must be defined in debian builder pods
+    for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do
+        if [[ -z "${!var}" ]] ; then
+            echo "$var must be defined in the environment!" >&2
+            exit 1
+        fi
    done
+    unset var
+
+    # Replace "@...@" tokens in apt template files
+    function replace_vars {
+        sed -e "s!@DEBIAN_SNAPSHOT@!${DEBIAN_SNAPSHOT}!g" \
+            -e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \
+            -e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \
+            -e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \
+            -e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \
+            "$@"
+    }
+
+    # create apt/ files for the docker file
+    mkdir -p "${BUILDDIR}/apt"
+
+    # debian.sources.list
+    replace_vars "${SRC_DOCKER_DIR}/apt/debian.sources.list.in" >"${BUILDDIR}/apt/debian.sources.list"
+
+    # stx.sources: if user provided any --repo's use them instead of the template
+    if [[ "${#REPO_LIST[@]}" -gt 0 ]] ; then
+        rm -f "${BUILDDIR}/apt/stx.sources.list"
+        for repo in "${REPO_LIST[@]}" ; do
+            echo "$repo" >>"${BUILDDIR}/apt/stx.sources.list"
+        done
+        unset repo
+    # otherwise use the template file
+    else
+        replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list"
+    fi
+
+    # preferences: instantiate template once for every host in stx.sources.list
+    unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u)
+    echo -n >"${BUILDDIR}/apt/stx.preferences"
+    for host in $unique_hosts ; do
+        REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences"
+        echo >>"${BUILDDIR}/apt/stx.preferences"
+    done
+    unset host unique_hosts
+    unset -f replace_vars
 fi

 # Check to see if the OS image is already pulled
--- a/build-tools/build-docker-images/stx-debian/Dockerfile.stable
+++ b/build-tools/build-docker-images/stx-debian/Dockerfile.stable
@ -1,11 +1,34 @@
-# Expected build arguments:
-#   RELEASE: debian release
-#
-ARG RELEASE=11.3
+# Start with an the old-ish bullseye release (11.2), then upgrade --
+# to make sure packages that come pre-installed in the debian:XXX image
+# are older than anything in StarlingX.
+ARG RELEASE=11.2
 FROM debian:${RELEASE}

 ENV DEBIAN_FRONTEND=noninteractive

+# Install latest ca-certificates
+RUN apt-get -y update && \
+    apt-get -y --no-install-recommends --no-install-suggests install ca-certificates
+
+# Disable upstream debian repos
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.disabled
+
+# Install apt repos
+COPY apt/debian.sources.list /etc/apt/sources.list.d/debian.list.disabled
+COPY apt/stx.sources.list /etc/apt/sources.list.d/stx.list.disabled
+COPY apt/stx.preferences /etc/apt/preferences.d/stx
+
+# Enable stx repo
+RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list
+
+# Clean apt cache
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Upgrade base packages to versions in the managed repos
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
 # repo templates:
 #   /etc/apt/sources.list.d/
 #     debian.list.disabled         - vanilla debian repos
@ -25,11 +48,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 # Enabling the upstream repos ("debian.list") is dangerous because it
 # may conflict with packages in stx.list.
 #
-COPY stx.apt.sources.list /etc/apt/sources.list.d/stx.list.disabled
-RUN mv /etc/apt/sources.list /etc/apt/sources.list.d/debian.list.disabled

-RUN cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \
-    apt-get update -y && \
+RUN apt-get update -y && \
    apt-get upgrade -y && \
    apt-get install -y \
 # FIXME: uncomment once qemu is ported to debian (starlingx/integ)
--- a/build-tools/build-docker-images/stx-debian/apt/debian.sources.list.in
+++ b/build-tools/build-docker-images/stx-debian/apt/debian.sources.list.in
@ -0,0 +1,2 @@
+deb [check-valid-until=0] @DEBIAN_SNAPSHOT@ @DEBIAN_DISTRIBUTION@ main
+deb [check-valid-until=0] @DEBIAN_SECURITY_SNAPSHOT@ @DEBIAN_DISTRIBUTION@-security main
--- a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in
+++ b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in
@ -0,0 +1,4 @@
+Explanation: Prefer StarlingX repos over vanilla Debian
+Package: *
+Pin: origin "@REPOMGR_HOST@"
+Pin-Priority: 999
--- a/build-tools/build-docker-images/stx-debian/apt/stx.sources.list.in
+++ b/build-tools/build-docker-images/stx-debian/apt/stx.sources.list.in
@ -0,0 +1,2 @@
+deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-binary @DEBIAN_DISTRIBUTION@ main
+deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-build @DEBIAN_DISTRIBUTION@ main
--- a/build-tools/build-wheels/build-base-wheels.sh
+++ b/build-tools/build-wheels/build-base-wheels.sh
@ -27,6 +27,7 @@ BUILD_STREAM=stable
 HTTP_PROXY=""
 HTTPS_PROXY=""
 NO_PROXY=""
+USE_DOCKER_CACHE=no
 : ${PYTHON3:=python3}
 declare -i MAX_ATTEMPTS=1

@ -46,10 +47,15 @@ Options:
    --stream:         Build stream, stable or dev (default: stable)
    --attempts:       Max attempts, in case of failure (default: 1)

+    --cache:          Allow docker to use filesystem cache when building
+                        CAUTION: this option may ignore locally-generated
+                                 packages and is meant for debugging the build
+                                 scripts.
+
 EOF
 }

-OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts: -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,cache -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -104,6 +110,10 @@ while true; do
            MAX_ATTEMPTS=$2
            shift 2
            ;;
+        --cache)
+            USE_DOCKER_CACHE=yes
+            shift
+            ;;
        -h | --help )
            usage
            exit 1
@ -297,13 +307,13 @@ mkdir -p "${DOCKER_BUILD_PATH}"
 # Replace "@...@" vars in apt/*.in files
 if [[ "${OS}" == "debian" ]] ; then
    (
-        # REPOMGR_DEPLOY_URL must be defined in the environment and refer
-        # to the k8s repomgr service. It is normally defined by the helm
-        # chart of STX tools.
-        if [[ -z "$REPOMGR_DEPLOY_URL" ]] ; then
-            echo "REPOMGR_DEPLOY_URL must be defined in the environment!" >&2
-            exit 1
-        fi
+        # These are normally defined by the helm chart of stx tools
+        for var in REPOMGR_DEPLOY_URL DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION ; do
+            if [[ -z "${!var}" ]] ; then
+                echo "$var must be defined in the environment!" >&2
+                exit 1
+            fi
+        done

        # Make sure pyhon3 exists
        $PYTHON3 --version >/dev/null || exit 1
@ -325,8 +335,11 @@ print (urlparse (sys.argv[1]).hostname)
        count=0
        for src in "${DOCKER_BUILD_PATH}/${OS}/apt"/*.in ; do
            dst="${src%.in}"
-            sed -e "s#@REPOMGR_URL@#$REPOMGR_DEPLOY_URL#g" \
+            sed -e "s#@REPOMGR_DEPLOY_URL@#$REPOMGR_DEPLOY_URL#g" \
                -e "s#@REPOMGR_HOST@#$REPOMGR_HOST#g" \
+                -e "s#@DEBIAN_SNAPSHOT@#$DEBIAN_SNAPSHOT#g" \
+                -e "s#@DEBIAN_SECURITY_SNAPSHOT@#$DEBIAN_SECURITY_SNAPSHOT#g" \
+                -e "s#@DEBIAN_DISTRIBUTION@#$DEBIAN_DISTRIBUTION#g" \
                "$src" >"$dst" || exit 1
            let ++count
        done
@ -357,6 +370,10 @@ if [ ! -z "$NO_PROXY" ]; then
    BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY)
 fi

+if [[ "$USE_DOCKER_CACHE" != "yes" ]] ; then
+    BUILD_ARGS+=("--no-cache")
+fi
+
 BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME})
 BUILD_ARGS+=(-f ${DOCKER_BUILD_PATH}/${OS}/Dockerfile ${DOCKER_BUILD_PATH})

--- a/build-tools/build-wheels/build-wheel-tarball.sh
+++ b/build-tools/build-wheels/build-wheel-tarball.sh
@ -31,6 +31,7 @@ KEEP_IMAGE=no
 DOCKER_USER=${USER}
 declare -i MAX_ATTEMPTS=1
 PYTHON2=no
+USE_DOCKER_CACHE=no

 # Requirement/constraint URLs -- these will be read from openstack.cfg
 STABLE_OPENSTACK_REQ_URL=
@ -75,10 +76,14 @@ Options:
    --python2:    Build a python2 tarball
    --keep-image: Don't delete wheel builder image at the end

+    --cache:      Allow docker to use filesystem cache when building
+                    CAUTION: this option may ignore locally-generated
+                             packages and is meant for debugging the build
+                             scripts.
 EOF
 }

-OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,keep-image -- "$@")
+OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,keep-image,cache -- "$@")
 if [ $? -ne 0 ]; then
    usage
    exit 1
@ -149,6 +154,10 @@ while true; do
            KEEP_IMAGE=yes
            shift
            ;;
+        --cache)
+            USE_DOCKER_CACHE=yes
+            shift
+            ;;
        -h | --help )
            usage
            exit 1
@ -214,6 +223,10 @@ if [ "$KEEP_IMAGE" = "yes" ]; then
    BUILD_BASE_WL_ARGS+=(--keep-image)
 fi

+if [[ "$USE_DOCKER_CACHE" == "yes" ]] ; then
+    BUILD_BASE_WL_ARGS+=(--cache)
+fi
+
 ${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS}
 if [ $? -ne 0 ]; then
    echo "Failure running build-base-wheels.sh" >&2
--- a/build-tools/build-wheels/debian/Dockerfile
+++ b/build-tools/build-wheels/debian/Dockerfile
@ -1,16 +1,39 @@
-ARG RELEASE=11.3
+# Start with an the old-ish bullseye release (11.2), then upgrade --
+# to make sure packages that come pre-installed in the debian:XXX image
+# are older than anything in StarlingX.
+ARG RELEASE=11.2
 FROM debian:${RELEASE}

+ENV DEBIAN_FRONTEND=noninteractive
+
 ARG BUILD_STREAM=stable

-# Install apt repos
-RUN mv /etc/apt/sources.list /etc/apt/sources.list.d/debian.list
-COPY debian/apt/sources.list /etc/apt/sources.list.d/stx.list
-COPY debian/apt/preferences /etc/apt/preferences.d/stx
+# Install latest ca-certificates
+RUN apt-get -y update && \
+    apt-get -y --no-install-recommends --no-install-suggests install ca-certificates

-# FIXME: disable upstream bullseye repo. Requires all dependent packages
+# Disable upstream debian repos
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.disabled
+
+# Install apt repos
+COPY debian/apt/debian.sources.list /etc/apt/sources.list.d/debian.list
+COPY debian/apt/stx.sources.list /etc/apt/sources.list.d/stx.list
+COPY debian/apt/stx.preferences /etc/apt/preferences.d/stx
+
+# Clean apt cache
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Upgrade base packages to versions in the managed repos
+RUN mv /etc/apt/sources.list.d/debian.list /etc/apt/sources.list.d/debian.list.disabled && \
+    apt-get -y update && \
+    apt-get -y upgrade && \
+    mv /etc/apt/sources.list.d/debian.list.disabled /etc/apt/sources.list.d/debian.list && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# FIXME: disable vanilla bullseye repo. Requires all dependent packages
 # referenced by apt-get to be added to stx-tools .lst file(s). Otherwise
 # we get a "random" set of packages, some from upstream, some from STX.
+# We may also get package conflicts between vanilla debian & STX repos.

 # FIXME: there's no ussuri/bullseye port, best we can do is "victoria" or more recent
 #RUN set -ex ; \
@ -24,7 +47,6 @@ RUN set -ex ; \
    apt-get -y update ; \
    apt-get -y --no-install-recommends --no-install-suggests install \
        bzip2 \
-        ca-certificates \
        g++ \
        gcc \
        git \
--- a/build-tools/build-wheels/debian/apt/debian.sources.list.in
+++ b/build-tools/build-wheels/debian/apt/debian.sources.list.in
@ -0,0 +1,2 @@
+deb [check-valid-until=0] @DEBIAN_SNAPSHOT@ @DEBIAN_DISTRIBUTION@ main
+deb [check-valid-until=0] @DEBIAN_SECURITY_SNAPSHOT@ @DEBIAN_DISTRIBUTION@-security main
--- a/build-tools/build-wheels/debian/apt/sources.list.in
+++ b/build-tools/build-wheels/debian/apt/sources.list.in
@ -1,2 +0,0 @@
-deb [trusted=yes] @REPOMGR_URL@/deb-local-binary bullseye main
-deb [trusted=yes] @REPOMGR_URL@/deb-local-build bullseye main
--- a/build-tools/build-wheels/debian/apt/stx.preferences.in
+++ b/build-tools/build-wheels/debian/apt/stx.preferences.in
--- a/build-tools/build-wheels/debian/apt/stx.sources.list.in
+++ b/build-tools/build-wheels/debian/apt/stx.sources.list.in
@ -0,0 +1,2 @@
+deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-binary @DEBIAN_DISTRIBUTION@ main
+deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-build @DEBIAN_DISTRIBUTION@ main