diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index eca7fb75..76bd8ca8 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -238,7 +238,8 @@ if [ ${VALID_OS} -ne 0 ]; then exit 1 fi -SRC_DOCKERFILE=${MY_SCRIPT_DIR}/stx-${OS}/Dockerfile.${BUILD_STREAM} +SRC_DOCKER_DIR="${MY_SCRIPT_DIR}/stx-${OS}" +SRC_DOCKERFILE="${SRC_DOCKER_DIR}"/Dockerfile.${BUILD_STREAM} if [[ -z "$OS_VERSION" ]]; then OS_VERSION=$( sed -n -r 's/^\s*ARG\s+RELEASE\s*=\s*([^ \t#]+).*/\1/ip' $SRC_DOCKERFILE | head -n 1 @@ -275,14 +276,8 @@ if [ ${#REPO_LIST[@]} -eq 0 ]; then if [[ "$OS" == "centos" ]] ; then REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS") REPO_LIST+=("stx-distro,http://${HOST}:8089${MY_REPO}/cgcs-${OS}-repo/Binary") - else - if [[ -z "$REPOMGR_DEPLOY_URL" ]] ; then - echo "Required env variable REPOMGR_DEPLOY_URL is not defined!" >&2 - exit 1 - fi - REPO_LIST+=("deb [trusted=yes] $REPOMGR_DEPLOY_URL/deb-local-binary bullseye main") - REPO_LIST+=("deb [trusted=yes] $REPOMGR_DEPLOY_URL/deb-local-build bullseye main") fi + # debian is handled down below elif [ "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then echo "Either --local or --repo must be specified" >&2 exit 1 @@ -336,11 +331,53 @@ EOF REPO_OPTS="${REPO_OPTS} --enablerepo=${repo_name}" done else - STX_APT_SOURCES_FILE=${BUILDDIR}/stx.apt.sources.list - rm -f "$STX_APT_SOURCES_FILE" - for repo in "${REPO_LIST[@]}" ; do - echo "$repo" >>"$STX_APT_SOURCES_FILE" + + # These env vars must be defined in debian builder pods + for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do + if [[ -z "${!var}" ]] ; then + echo "$var must be defined in the environment!" >&2 + exit 1 + fi done + unset var + + # Replace "@...@" tokens in apt template files + function replace_vars { + sed -e "s!@DEBIAN_SNAPSHOT@!${DEBIAN_SNAPSHOT}!g" \ + -e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \ + -e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \ + -e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \ + -e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \ + "$@" + } + + # create apt/ files for the docker file + mkdir -p "${BUILDDIR}/apt" + + # debian.sources.list + replace_vars "${SRC_DOCKER_DIR}/apt/debian.sources.list.in" >"${BUILDDIR}/apt/debian.sources.list" + + # stx.sources: if user provided any --repo's use them instead of the template + if [[ "${#REPO_LIST[@]}" -gt 0 ]] ; then + rm -f "${BUILDDIR}/apt/stx.sources.list" + for repo in "${REPO_LIST[@]}" ; do + echo "$repo" >>"${BUILDDIR}/apt/stx.sources.list" + done + unset repo + # otherwise use the template file + else + replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list" + fi + + # preferences: instantiate template once for every host in stx.sources.list + unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u) + echo -n >"${BUILDDIR}/apt/stx.preferences" + for host in $unique_hosts ; do + REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences" + echo >>"${BUILDDIR}/apt/stx.preferences" + done + unset host unique_hosts + unset -f replace_vars fi # Check to see if the OS image is already pulled diff --git a/build-tools/build-docker-images/stx-debian/Dockerfile.stable b/build-tools/build-docker-images/stx-debian/Dockerfile.stable index 6a1669a4..5b0acfd7 100644 --- a/build-tools/build-docker-images/stx-debian/Dockerfile.stable +++ b/build-tools/build-docker-images/stx-debian/Dockerfile.stable @@ -1,11 +1,34 @@ -# Expected build arguments: -# RELEASE: debian release -# -ARG RELEASE=11.3 +# Start with an the old-ish bullseye release (11.2), then upgrade -- +# to make sure packages that come pre-installed in the debian:XXX image +# are older than anything in StarlingX. +ARG RELEASE=11.2 FROM debian:${RELEASE} ENV DEBIAN_FRONTEND=noninteractive +# Install latest ca-certificates +RUN apt-get -y update && \ + apt-get -y --no-install-recommends --no-install-suggests install ca-certificates + +# Disable upstream debian repos +RUN mv /etc/apt/sources.list /etc/apt/sources.list.disabled + +# Install apt repos +COPY apt/debian.sources.list /etc/apt/sources.list.d/debian.list.disabled +COPY apt/stx.sources.list /etc/apt/sources.list.d/stx.list.disabled +COPY apt/stx.preferences /etc/apt/preferences.d/stx + +# Enable stx repo +RUN cp /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list + +# Clean apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Upgrade base packages to versions in the managed repos +RUN apt-get -y update && \ + apt-get -y upgrade && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + # repo templates: # /etc/apt/sources.list.d/ # debian.list.disabled - vanilla debian repos @@ -25,11 +48,8 @@ ENV DEBIAN_FRONTEND=noninteractive # Enabling the upstream repos ("debian.list") is dangerous because it # may conflict with packages in stx.list. # -COPY stx.apt.sources.list /etc/apt/sources.list.d/stx.list.disabled -RUN mv /etc/apt/sources.list /etc/apt/sources.list.d/debian.list.disabled -RUN cp -f /etc/apt/sources.list.d/stx.list.disabled /etc/apt/sources.list.d/stx.list && \ - apt-get update -y && \ +RUN apt-get update -y && \ apt-get upgrade -y && \ apt-get install -y \ # FIXME: uncomment once qemu is ported to debian (starlingx/integ) diff --git a/build-tools/build-docker-images/stx-debian/apt/debian.sources.list.in b/build-tools/build-docker-images/stx-debian/apt/debian.sources.list.in new file mode 100644 index 00000000..1b087a5c --- /dev/null +++ b/build-tools/build-docker-images/stx-debian/apt/debian.sources.list.in @@ -0,0 +1,2 @@ +deb [check-valid-until=0] @DEBIAN_SNAPSHOT@ @DEBIAN_DISTRIBUTION@ main +deb [check-valid-until=0] @DEBIAN_SECURITY_SNAPSHOT@ @DEBIAN_DISTRIBUTION@-security main diff --git a/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in new file mode 100644 index 00000000..5863a2e0 --- /dev/null +++ b/build-tools/build-docker-images/stx-debian/apt/stx.preferences.part.in @@ -0,0 +1,4 @@ +Explanation: Prefer StarlingX repos over vanilla Debian +Package: * +Pin: origin "@REPOMGR_HOST@" +Pin-Priority: 999 diff --git a/build-tools/build-docker-images/stx-debian/apt/stx.sources.list.in b/build-tools/build-docker-images/stx-debian/apt/stx.sources.list.in new file mode 100644 index 00000000..50f14804 --- /dev/null +++ b/build-tools/build-docker-images/stx-debian/apt/stx.sources.list.in @@ -0,0 +1,2 @@ +deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-binary @DEBIAN_DISTRIBUTION@ main +deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-build @DEBIAN_DISTRIBUTION@ main diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index 5ee4248c..899e82ef 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -27,6 +27,7 @@ BUILD_STREAM=stable HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY="" +USE_DOCKER_CACHE=no : ${PYTHON3:=python3} declare -i MAX_ATTEMPTS=1 @@ -46,10 +47,15 @@ Options: --stream: Build stream, stable or dev (default: stable) --attempts: Max attempts, in case of failure (default: 1) + --cache: Allow docker to use filesystem cache when building + CAUTION: this option may ignore locally-generated + packages and is meant for debugging the build + scripts. + EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release:,stream:,http_proxy:,https_proxy:,no_proxy:,attempts:,cache -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -104,6 +110,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --cache) + USE_DOCKER_CACHE=yes + shift + ;; -h | --help ) usage exit 1 @@ -297,13 +307,13 @@ mkdir -p "${DOCKER_BUILD_PATH}" # Replace "@...@" vars in apt/*.in files if [[ "${OS}" == "debian" ]] ; then ( - # REPOMGR_DEPLOY_URL must be defined in the environment and refer - # to the k8s repomgr service. It is normally defined by the helm - # chart of STX tools. - if [[ -z "$REPOMGR_DEPLOY_URL" ]] ; then - echo "REPOMGR_DEPLOY_URL must be defined in the environment!" >&2 - exit 1 - fi + # These are normally defined by the helm chart of stx tools + for var in REPOMGR_DEPLOY_URL DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION ; do + if [[ -z "${!var}" ]] ; then + echo "$var must be defined in the environment!" >&2 + exit 1 + fi + done # Make sure pyhon3 exists $PYTHON3 --version >/dev/null || exit 1 @@ -325,8 +335,11 @@ print (urlparse (sys.argv[1]).hostname) count=0 for src in "${DOCKER_BUILD_PATH}/${OS}/apt"/*.in ; do dst="${src%.in}" - sed -e "s#@REPOMGR_URL@#$REPOMGR_DEPLOY_URL#g" \ + sed -e "s#@REPOMGR_DEPLOY_URL@#$REPOMGR_DEPLOY_URL#g" \ -e "s#@REPOMGR_HOST@#$REPOMGR_HOST#g" \ + -e "s#@DEBIAN_SNAPSHOT@#$DEBIAN_SNAPSHOT#g" \ + -e "s#@DEBIAN_SECURITY_SNAPSHOT@#$DEBIAN_SECURITY_SNAPSHOT#g" \ + -e "s#@DEBIAN_DISTRIBUTION@#$DEBIAN_DISTRIBUTION#g" \ "$src" >"$dst" || exit 1 let ++count done @@ -357,6 +370,10 @@ if [ ! -z "$NO_PROXY" ]; then BUILD_ARGS+=(--build-arg no_proxy=$NO_PROXY) fi +if [[ "$USE_DOCKER_CACHE" != "yes" ]] ; then + BUILD_ARGS+=("--no-cache") +fi + BUILD_ARGS+=(-t ${BUILD_IMAGE_NAME}) BUILD_ARGS+=(-f ${DOCKER_BUILD_PATH}/${OS}/Dockerfile ${DOCKER_BUILD_PATH}) diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index c710501c..ca398894 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -31,6 +31,7 @@ KEEP_IMAGE=no DOCKER_USER=${USER} declare -i MAX_ATTEMPTS=1 PYTHON2=no +USE_DOCKER_CACHE=no # Requirement/constraint URLs -- these will be read from openstack.cfg STABLE_OPENSTACK_REQ_URL= @@ -75,10 +76,14 @@ Options: --python2: Build a python2 tarball --keep-image: Don't delete wheel builder image at the end + --cache: Allow docker to use filesystem cache when building + CAUTION: this option may ignore locally-generated + packages and is meant for debugging the build + scripts. EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,keep-image -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,stream:,http_proxy:,https_proxy:,no_proxy:,version:,attempts:,python2,keep-image,cache -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -149,6 +154,10 @@ while true; do KEEP_IMAGE=yes shift ;; + --cache) + USE_DOCKER_CACHE=yes + shift + ;; -h | --help ) usage exit 1 @@ -214,6 +223,10 @@ if [ "$KEEP_IMAGE" = "yes" ]; then BUILD_BASE_WL_ARGS+=(--keep-image) fi +if [[ "$USE_DOCKER_CACHE" == "yes" ]] ; then + BUILD_BASE_WL_ARGS+=(--cache) +fi + ${MY_SCRIPT_DIR}/build-base-wheels.sh ${BUILD_BASE_WL_ARGS[@]} --attempts ${MAX_ATTEMPTS} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 diff --git a/build-tools/build-wheels/debian/Dockerfile b/build-tools/build-wheels/debian/Dockerfile index c3dd5012..5b8f10f6 100644 --- a/build-tools/build-wheels/debian/Dockerfile +++ b/build-tools/build-wheels/debian/Dockerfile @@ -1,16 +1,39 @@ -ARG RELEASE=11.3 +# Start with an the old-ish bullseye release (11.2), then upgrade -- +# to make sure packages that come pre-installed in the debian:XXX image +# are older than anything in StarlingX. +ARG RELEASE=11.2 FROM debian:${RELEASE} +ENV DEBIAN_FRONTEND=noninteractive + ARG BUILD_STREAM=stable -# Install apt repos -RUN mv /etc/apt/sources.list /etc/apt/sources.list.d/debian.list -COPY debian/apt/sources.list /etc/apt/sources.list.d/stx.list -COPY debian/apt/preferences /etc/apt/preferences.d/stx +# Install latest ca-certificates +RUN apt-get -y update && \ + apt-get -y --no-install-recommends --no-install-suggests install ca-certificates -# FIXME: disable upstream bullseye repo. Requires all dependent packages +# Disable upstream debian repos +RUN mv /etc/apt/sources.list /etc/apt/sources.list.disabled + +# Install apt repos +COPY debian/apt/debian.sources.list /etc/apt/sources.list.d/debian.list +COPY debian/apt/stx.sources.list /etc/apt/sources.list.d/stx.list +COPY debian/apt/stx.preferences /etc/apt/preferences.d/stx + +# Clean apt cache +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Upgrade base packages to versions in the managed repos +RUN mv /etc/apt/sources.list.d/debian.list /etc/apt/sources.list.d/debian.list.disabled && \ + apt-get -y update && \ + apt-get -y upgrade && \ + mv /etc/apt/sources.list.d/debian.list.disabled /etc/apt/sources.list.d/debian.list && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# FIXME: disable vanilla bullseye repo. Requires all dependent packages # referenced by apt-get to be added to stx-tools .lst file(s). Otherwise # we get a "random" set of packages, some from upstream, some from STX. +# We may also get package conflicts between vanilla debian & STX repos. # FIXME: there's no ussuri/bullseye port, best we can do is "victoria" or more recent #RUN set -ex ; \ @@ -24,7 +47,6 @@ RUN set -ex ; \ apt-get -y update ; \ apt-get -y --no-install-recommends --no-install-suggests install \ bzip2 \ - ca-certificates \ g++ \ gcc \ git \ diff --git a/build-tools/build-wheels/debian/apt/debian.sources.list.in b/build-tools/build-wheels/debian/apt/debian.sources.list.in new file mode 100644 index 00000000..1b087a5c --- /dev/null +++ b/build-tools/build-wheels/debian/apt/debian.sources.list.in @@ -0,0 +1,2 @@ +deb [check-valid-until=0] @DEBIAN_SNAPSHOT@ @DEBIAN_DISTRIBUTION@ main +deb [check-valid-until=0] @DEBIAN_SECURITY_SNAPSHOT@ @DEBIAN_DISTRIBUTION@-security main diff --git a/build-tools/build-wheels/debian/apt/sources.list.in b/build-tools/build-wheels/debian/apt/sources.list.in deleted file mode 100644 index 7b7f6027..00000000 --- a/build-tools/build-wheels/debian/apt/sources.list.in +++ /dev/null @@ -1,2 +0,0 @@ -deb [trusted=yes] @REPOMGR_URL@/deb-local-binary bullseye main -deb [trusted=yes] @REPOMGR_URL@/deb-local-build bullseye main diff --git a/build-tools/build-wheels/debian/apt/preferences.in b/build-tools/build-wheels/debian/apt/stx.preferences.in similarity index 100% rename from build-tools/build-wheels/debian/apt/preferences.in rename to build-tools/build-wheels/debian/apt/stx.preferences.in diff --git a/build-tools/build-wheels/debian/apt/stx.sources.list.in b/build-tools/build-wheels/debian/apt/stx.sources.list.in new file mode 100644 index 00000000..50f14804 --- /dev/null +++ b/build-tools/build-wheels/debian/apt/stx.sources.list.in @@ -0,0 +1,2 @@ +deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-binary @DEBIAN_DISTRIBUTION@ main +deb [trusted=yes] @REPOMGR_DEPLOY_URL@/deb-local-build @DEBIAN_DISTRIBUTION@ main