starlingx/root
Code Issues Proposed changes
root/build-tools/build-docker-images/build-stx-base.sh
Davlet Panech 30d4eb3566 debian: use debian snapshot repos in docker apps 
- build-wheels, build-docker-images: add command line options to enable
  the use of docker filesystem cache when building
- use snapshot repo URLs defined by tools in build pods' environment to
  generate apt sources list files. These are defined in stx.conf.

TESTS
===============================
All tests were performed on Debian:
- Build wheels
- Build base image
- Build a samnple docker app that derives from the base image

Story: 2009897
Task: 45185

Depends-On: https://review.opendev.org/c/starlingx/tools/+/839396
Change-Id: I58a1ee002bb8161c492b3bcf8cd4bbcb6b4fcae4
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
2022-04-26 11:08:30 -04:00

467 lines
13 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
#
# Copyright (c) 2018-2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This utility builds the StarlingX base image
#
MY_SCRIPT_DIR=$(dirname $(readlink -f $0))
source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
exit 1
fi
SUPPORTED_OS_ARGS=('centos' 'debian')
OS= # default: autodetect
OS_VERSION= # default: lookup "ARG RELEASE" in Dockerfile
BUILD_STREAM=stable
IMAGE_VERSION=
PUSH=no
PROXY=""
CONFIG_FILE=""
DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images"
DEFAULT_CONFIG_FILE_PREFIX="base-image-build"
DOCKER_USER=${USER}
DOCKER_REGISTRY=
declare -a REPO_LIST
REPO_OPTS=
LOCAL=no
CLEAN=no
TAG_LATEST=no
LATEST_TAG=latest
HOST=${HOSTNAME}
USE_DOCKER_CACHE=no
declare -i MAX_ATTEMPTS=1
function usage {
cat >&2 <<EOF
Usage:
$(basename $0)
Options:
--os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]})
--os-version: Specify OS version
--version: Specify version for output image
--stream: Build stream, stable or dev (default: stable)
--repo: Software repository, can be specified multiple times
* CentOS format: "NAME,BASEURL"
* Debian format: "TYPE [OPTION=VALUE...] URL DISTRO COMPONENTS..."
This will be added to /etc/apt/sources.list as is,
see also sources.list(5) manpage.
--local: Use local build for software repository (cannot be used with --repo)
--push: Push to docker repo
--proxy: Set proxy <URL>:<PORT>
--latest: Add a 'latest' tag when pushing
--latest-tag: Use the provided tag when pushing latest.
--user: Docker repo userid
--registry: Docker registry
--clean: Remove image(s) from local registry
--hostname: build repo host
--attempts: Max attempts, in case of failure (default: 1)
--config-file:Specify a path to a config file which will specify additional arguments to be passed into the command
--cache: Allow docker to use cached filesystem layers when building
CAUTION: this option may ignore locally-generated packages
and is meant for debugging the build scripts.
EOF
}
function get_args_from_file {
# get additional args from specified file.
local line key value
echo "Get args from file: $1"
while read line
do
# skip comments & empty lines
if echo "$line" | grep -q -E '^\s*(#.*)?$' ; then
continue
fi
key="${line%%=*}"
value=${line#*=}
echo "--$key '$value'"
case "$key" in
version)
if [ -z "${IMAGE_VERSION}" ]; then
IMAGE_VERSION="$value"
fi
;;
user)
if [ -z "${DOCKER_USER}" ]; then
DOCKER_USER="$value"
fi
;;
proxy)
if [ -z "${PROXY}" ]; then
PROXY="$value"
fi
;;
registry)
if [ -z "${DOCKER_REGISTRY}" ]; then
# Add a trailing / if needed
DOCKER_REGISTRY="${value%/}/"
fi
;;
repo)
REPO_LIST+=("$value")
;;
*)
echo "WARNING: $line: ignoring unknown option \"$key\"" >&2
;;
esac
done <"$1"
}
OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,cache,hostname:,attempts:,config-file: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
fi
eval set -- "${OPTS}"
while true; do
case $1 in
--)
# End of getopt arguments
shift
break
;;
--os)
OS=$2
shift 2
;;
--os-version)
OS_VERSION=$2
shift 2
;;
--version)
IMAGE_VERSION=$2
shift 2
;;
--stream)
BUILD_STREAM=$2
shift 2
;;
--release) # Temporarily keep --release support as an alias for --stream
BUILD_STREAM=$2
shift 2
;;
--repo)
REPO_LIST+=($2)
shift 2
;;
--local)
LOCAL=yes
shift
;;
--push)
PUSH=yes
shift
;;
--proxy)
PROXY=$2
shift 2
;;
--latest)
TAG_LATEST=yes
shift
;;
--latest-tag)
LATEST_TAG=$2
shift 2
;;
--user)
DOCKER_USER=$2
shift 2
;;
--registry)
# Add a trailing / if needed
DOCKER_REGISTRY="${2%/}/"
shift 2
;;
--clean)
CLEAN=yes
shift
;;
--cache)
USE_DOCKER_CACHE=yes
shift
;;
--hostname)
HOST=$2
shift 2
;;
--attempts)
MAX_ATTEMPTS=$2
shift 2
;;
--config-file)
CONFIG_FILE=$2
shift 2
;;
-h | --help )
usage
exit 1
;;
*)
usage
exit 1
;;
esac
done
# Validate the OS option
if [ -z "$OS" ] ; then
OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)"
if [[ -z "$OS" ]] ; then
echo "Unable to determine OS, please re-run with \`--os' option" >&2
exit 1
fi
fi
VALID_OS=1
for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
if [ "$OS" = "${supported_os}" ]; then
VALID_OS=0
break
fi
done
if [ ${VALID_OS} -ne 0 ]; then
echo "Unsupported OS specified: ${OS}" >&2
echo "Supported OS options: ${SUPPORTED_OS_ARGS[@]}" >&2
exit 1
fi
SRC_DOCKER_DIR="${MY_SCRIPT_DIR}/stx-${OS}"
SRC_DOCKERFILE="${SRC_DOCKER_DIR}"/Dockerfile.${BUILD_STREAM}
if [[ -z "$OS_VERSION" ]]; then
OS_VERSION=$(
sed -n -r 's/^\s*ARG\s+RELEASE\s*=\s*([^ \t#]+).*/\1/ip' $SRC_DOCKERFILE | head -n 1
[[ ${PIPESTATUS[0]} -eq 0 ]]
)
if [[ -z "$OS_VERSION" ]] ; then
echo "$SRC_DOCKERFILE: failed to determine OS_VERSION" >&2
exit 1
fi
fi
if [ -z "${IMAGE_VERSION}" ]; then
IMAGE_VERSION=${OS_VERSION}
fi
DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg"
# Read additional auguments from config file if it exists.
if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then
CONFIG_FILE=${DEFAULT_CONFIG_FILE}
fi
if [[ ! -z ${CONFIG_FILE} ]]; then
if [[ -f ${CONFIG_FILE} ]]; then
get_args_from_file ${CONFIG_FILE}
else
echo "Config file not found: ${CONFIG_FILE}"
exit 1
fi
fi
if [ ${#REPO_LIST[@]} -eq 0 ]; then
# Either --repo or --local must be specified
if [ "${LOCAL}" = "yes" ]; then
if [[ "$OS" == "centos" ]] ; then
REPO_LIST+=("local-std,http://${HOST}:8088${MY_WORKSPACE}/std/rpmbuild/RPMS")
REPO_LIST+=("stx-distro,http://${HOST}:8089${MY_REPO}/cgcs-${OS}-repo/Binary")
fi
# debian is handled down below
elif [ "${BUILD_STREAM}" != "dev" -a "${BUILD_STREAM}" != "master" ]; then
echo "Either --local or --repo must be specified" >&2
exit 1
fi
else
if [ "${LOCAL}" = "yes" ]; then
echo "Cannot specify both --local and --repo" >&2
exit 1
fi
fi
BUILDDIR=${MY_WORKSPACE}/std/build-images/stx-${OS}
if [ -d ${BUILDDIR} ]; then
# Leftover from previous build
rm -rf ${BUILDDIR}
fi
mkdir -p ${BUILDDIR}
if [ $? -ne 0 ]; then
echo "Failed to create ${BUILDDIR}" >&2
exit 1
fi
# Get the Dockerfile
cp ${SRC_DOCKERFILE} ${BUILDDIR}/Dockerfile
# Generate the stx.repo file
if [[ "$OS" == "centos" ]] ; then
STX_REPO_FILE=${BUILDDIR}/stx.repo
for repo in ${REPO_LIST[@]}; do
repo_name=$(echo $repo | awk -F, '{print $1}')
repo_baseurl=$(echo $repo | awk -F, '{print $2}')
if [ -z "${repo_name}" -o -z "${repo_baseurl}" ]; then
echo "Invalid repo specified: ${repo}" >&2
echo "Expected format: name,baseurl" >&2
exit 1
fi
cat >>${STX_REPO_FILE} <<EOF
[${repo_name}]
name=${repo_name}
baseurl=${repo_baseurl}
enabled=1
gpgcheck=0
skip_if_unavailable=1
metadata_expire=0
EOF
REPO_OPTS="${REPO_OPTS} --enablerepo=${repo_name}"
done
else
# These env vars must be defined in debian builder pods
for var in DEBIAN_SNAPSHOT DEBIAN_SECURITY_SNAPSHOT DEBIAN_DISTRIBUTION REPOMGR_DEPLOY_URL ; do
if [[ -z "${!var}" ]] ; then
echo "$var must be defined in the environment!" >&2
exit 1
fi
done
unset var
# Replace "@...@" tokens in apt template files
function replace_vars {
sed -e "s!@DEBIAN_SNAPSHOT@!${DEBIAN_SNAPSHOT}!g" \
-e "s!@DEBIAN_SECURITY_SNAPSHOT@!${DEBIAN_SECURITY_SNAPSHOT}!g" \
-e "s!@DEBIAN_DISTRIBUTION@!${DEBIAN_DISTRIBUTION}!g" \
-e "s!@REPOMGR_DEPLOY_URL@!${REPOMGR_DEPLOY_URL}!g" \
-e "s!@REPOMGR_HOST@!${REPOMGR_HOST}!g" \
"$@"
}
# create apt/ files for the docker file
mkdir -p "${BUILDDIR}/apt"
# debian.sources.list
replace_vars "${SRC_DOCKER_DIR}/apt/debian.sources.list.in" >"${BUILDDIR}/apt/debian.sources.list"
# stx.sources: if user provided any --repo's use them instead of the template
if [[ "${#REPO_LIST[@]}" -gt 0 ]] ; then
rm -f "${BUILDDIR}/apt/stx.sources.list"
for repo in "${REPO_LIST[@]}" ; do
echo "$repo" >>"${BUILDDIR}/apt/stx.sources.list"
done
unset repo
# otherwise use the template file
else
replace_vars "${SRC_DOCKER_DIR}/apt/stx.sources.list.in" >"${BUILDDIR}/apt/stx.sources.list"
fi
# preferences: instantiate template once for every host in stx.sources.list
unique_hosts=$(\grep -v -E '^\s*(#.*)?$' "${BUILDDIR}/apt/stx.sources.list" | sed -n -r 's#.*(https?|ftp)://([^/:[:space:]]+).*#\2#p' | sort -u)
echo -n >"${BUILDDIR}/apt/stx.preferences"
for host in $unique_hosts ; do
REPOMGR_HOST="$host" replace_vars "${SRC_DOCKER_DIR}/apt/stx.preferences.part.in" >>"${BUILDDIR}/apt/stx.preferences"
echo >>"${BUILDDIR}/apt/stx.preferences"
done
unset host unique_hosts
unset -f replace_vars
fi
# Check to see if the OS image is already pulled
docker images --format '{{.Repository}}:{{.Tag}}' ${OS}:${OS_VERSION} | grep -q "^${OS}:${OS_VERSION}$"
BASE_IMAGE_PRESENT=$?
# Pull the image anyway, to ensure it is up to date
docker pull ${OS}:${OS_VERSION}
# Build the image
IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION}
IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG}
declare -a BUILD_ARGS
BUILD_ARGS+=(--build-arg RELEASE=${OS_VERSION})
if [[ "$OS" == "centos" ]] ; then
BUILD_ARGS+=(--build-arg "REPO_OPTS=${REPO_OPTS}")
fi
# Add proxy to docker build
if [ ! -z "$PROXY" ]; then
BUILD_ARGS+=(--build-arg http_proxy=$PROXY)
fi
# Don't use docker cache
if [[ "$USE_DOCKER_CACHE" != "yes" ]] ; then
BUILD_ARGS+=("--no-cache")
fi
BUILD_ARGS+=(--tag ${IMAGE_NAME} ${BUILDDIR})
# Build base image
with_retries ${MAX_ATTEMPTS} docker build "${BUILD_ARGS[@]}"
if [ $? -ne 0 ]; then
echo "Failed running docker build command" >&2
exit 1
fi
if [ "${PUSH}" = "yes" ]; then
# Push the image
echo "Pushing image: ${IMAGE_NAME}"
docker push ${IMAGE_NAME}
if [ $? -ne 0 ]; then
echo "Failed running docker push command" >&2
exit 1
fi
if [ "$TAG_LATEST" = "yes" ]; then
docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST}
echo "Pushing image: ${IMAGE_NAME_LATEST}"
docker push ${IMAGE_NAME_LATEST}
if [ $? -ne 0 ]; then
echo "Failed running docker push command on latest" >&2
exit 1
fi
fi
fi
if [ "${CLEAN}" = "yes" ]; then
# Delete the images
echo "Deleting image: ${IMAGE_NAME}"
docker image rm ${IMAGE_NAME}
if [ $? -ne 0 ]; then
echo "Failed running docker image rm command" >&2
exit 1
fi
if [ "$TAG_LATEST" = "yes" ]; then
echo "Deleting image: ${IMAGE_NAME_LATEST}"
docker image rm ${IMAGE_NAME_LATEST}
if [ $? -ne 0 ]; then
echo "Failed running docker image rm command" >&2
exit 1
fi
fi
if [ ${BASE_IMAGE_PRESENT} -ne 0 ]; then
# The base image was not already present, so delete it
echo "Removing docker image ${OS}:${OS_VERSION}"
docker image rm ${OS}:${OS_VERSION}
if [ $? -ne 0 ]; then
echo "Failed to delete base image from docker" >&2
fi
fi
fi
View Git Blame Copy Permalink