From 6810ca2dfab7757ac3addfe5e85ff24687248ac8 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Mon, 27 May 2019 12:55:37 -0400 Subject: [PATCH] Tool for incremental image updates This update introduces a tool for incrementally updating images, update-stx-image.sh. This allows for small customizations to images, such as updating specific python modules. Additionally, this update corrects how docker-build-wheel.sh checks out a specific git branch, fetching the branch and checking out the FETCH_HEAD, where previously it was just checking out the branch. Change-Id: I09c1b2d7a3664a3af5fd887087522b1186eb1ddd Story: 2005248 Task: 33440 Signed-off-by: Don Penney --- .../internal-update-stx-image.sh | 121 +++++ .../build-docker-images/update-stx-image.sh | 488 ++++++++++++++++++ .../build-wheels/docker/docker-build-wheel.sh | 11 +- 3 files changed, 618 insertions(+), 2 deletions(-) create mode 100755 build-tools/build-docker-images/internal-update-stx-image.sh create mode 100755 build-tools/build-docker-images/update-stx-image.sh diff --git a/build-tools/build-docker-images/internal-update-stx-image.sh b/build-tools/build-docker-images/internal-update-stx-image.sh new file mode 100755 index 00000000..549837a1 --- /dev/null +++ b/build-tools/build-docker-images/internal-update-stx-image.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This runs inside a container to update the image +# + +UPDATES_DIR=/image-update +PIP_PACKAGES_DIR=${UPDATES_DIR}/pip-packages +DIST_PACKAGES_DIR=${UPDATES_DIR}/dist-packages +CUSTOMIZATION_SCRIPT=${UPDATES_DIR}/customize.sh + +OS_NAME=$(source /etc/os-release && echo ${NAME}) + +OPTS=$(getopt -o h -l help: -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +function usage { + cat >&2 <&2 + exit 1 + fi +} + +function install_dist_packages { + local -i file_count=0 + + file_count=$(find ${DIST_PACKAGES_DIR} -type f 2>/dev/null | wc -l) + + if [ ${file_count} -eq 0 ]; then + # No files, nothing to do + return 0 + fi + + case ${OS_NAME} in + "CentOS Linux") + install_centos_dist_packages + ;; + *) + echo "Unsupported OS for DIST_PACKAGES: ${OS_NAME}" >&2 + exit 1 + ;; + esac +} + +function install_pip_packages { + local modules + local wheels + modules=$(find ${PIP_PACKAGES_DIR}/modules/* -maxdepth 0 -type d 2>/dev/null) + wheels=$(find ${PIP_PACKAGES_DIR}/wheels/ -type f -name '*.whl' 2>/dev/null) + + if [ -z "${modules}" -a -z "${wheels}" ]; then + # Nothing to do + return 0 + fi + + pip install -vvv --no-deps --no-index --pre --no-cache-dir --only-binary :all: --no-compile --force-reinstall \ + ${modules} ${wheels} + + if [ $? -ne 0 ]; then + echo "Failed pip install" >&2 + exit 1 + fi +} + +function run_customization_script { + if [ -x "${CUSTOMIZATION_SCRIPT}" ]; then + bash -x ${CUSTOMIZATION_SCRIPT} + + if [ $? -ne 0 ]; then + echo "Failed customization script" >&2 + exit 1 + fi + fi +} + +# Update the image +install_dist_packages +install_pip_packages +run_customization_script + +exit 0 + diff --git a/build-tools/build-docker-images/update-stx-image.sh b/build-tools/build-docker-images/update-stx-image.sh new file mode 100755 index 00000000..8a27573c --- /dev/null +++ b/build-tools/build-docker-images/update-stx-image.sh @@ -0,0 +1,488 @@ +#!/bin/bash +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Utility for incremental updates to an image +# + +MY_SCRIPT_DIR=$(dirname $(readlink -f $0)) + +source ${MY_SCRIPT_DIR}/../build-wheels/utils.sh + +# Required env vars +if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then + echo "Environment not setup for builds" >&2 + exit 1 +fi + +PUSH=no +PROXY="" +DOCKER_USER= +DOCKER_REGISTRY= +FILE_BASEDIR=${PWD} +FROM= +CLEAN=no +DIST_PACKAGES= +CUSTOMIZATION_SCRIPT= +UPDATE_ID="unnamed-update" +declare -i IMAGE_UPDATE_VER= +declare -a WHEELS +declare -a DIST_PACKAGES +declare -a MODULE_SRC +declare -a EXTRA_FILES +declare -i MAX_ATTEMPTS=1 + + +function usage { + cat >&2 <: + --user: Docker repo userid + --registry: Docker registry + --clean: Remove image(s) from local registry + --attempts: Max attempts, in case of failure (default: 1) + --update-id: Update ID + + +EOF +} + +function copy_files_to_workdir { + # + # Utility function to copy files to the workdir + # + local destdir=$1 + shift + + if [ ${#@} -le 0 ]; then + # No files in list, nothing to do + return 0 + fi + + mkdir -p ${destdir} + if [ $? -ne 0 ]; then + echo "Failed to create dir: ${destdir}" >&2 + exit 1 + fi + + for f in $*; do + if [[ ${f} =~ ^(http|https|git): ]]; then + pushd ${destdir} + with_retries ${MAX_ATTEMPTS} wget ${f} + if [ $? -ne 0 ]; then + echo "Failed to download $f to ${destdir}" >&2 + exit 1 + fi + else + cp -v ${f} ${destdir}/ + if [ $? -ne 0 ]; then + echo "Failed to copy files to ${destdir}" >&2 + exit 1 + fi + fi + done +} + +function hardcode_python_module_version { + # + # Update a python module's setup.py to hardcode the version, + # allowing for pip to read the version without git installed + # inside the container. + # + local module_dir=$1 + local module_ver=$2 + + if [ ! -f ${module_dir}/setup.py ]; then + # Nothing to do + return 0 + fi + + pushd ${module_dir} + grep -q 'pbr=True' ./setup.py + if [ $? -eq 0 ]; then + if [ -z "${module_ver}" ]; then + # Get the calculated version + module_ver=$(python ./setup.py --version) + fi + chmod u+w ./setup.py # just in case + sed -i "s/pbr=True/version='${module_ver}'/" ./setup.py + else + # This function can be extended in the future to support + # hardcoding/updating the version in modules that don't + # use PBR, if required. + echo "Module ($(basename ${module_dir})) does not have pbr=True." >&2 + echo "Skipping updating version in code." >&2 + fi + popd +} + +function update_image_record { + # Update the image record file with a new/updated entry + local LABEL=$1 + local TAG=$2 + local FILE=$3 + + touch ${FILE} + + grep -q "/${LABEL}:" ${FILE} + if [ $? -eq 0 ]; then + # Update the existing record + sed -i "s#.*/${LABEL}:.*#${TAG}#" ${FILE} + else + # Add a new record + echo "${TAG}" >> ${FILE} + fi +} + +function read_params_from_file { + local FILE=$1 + + if [ ! -f "${FILE}" ]; then + echo "Specified file does not exist: ${FILE}" >&2 + exit 1 + fi + + # Get parameters from file + # + # To avoid polluting the environment and impacting + # other builds, we're going to explicitly grab specific + # variables from the directives file. While this does + # mean the file is sourced repeatedly, it ensures we + # don't get junk. + FROM=$(source ${FILE} && echo ${FROM}) + IMAGE_UPDATE_VER=$(source ${FILE} && echo ${IMAGE_UPDATE_VER}) + CUSTOMIZATION_SCRIPT=$(source ${FILE} && echo ${CUSTOMIZATION_SCRIPT}) + + WHEELS=($(source ${FILE} && echo ${WHEELS})) + DIST_PACKAGES=($(source ${FILE} && echo ${DIST_PACKAGES})) + MODULE_SRC=($(source ${FILE} && echo ${MODULE_SRC})) + EXTRA_FILES=($(source ${FILE} && echo ${EXTRA_FILES})) + + FILE_BASEDIR=$(dirname ${FILE}) +} + +OPTS=$(getopt -o h -l help,file:,from:,wheel:,module-src:,pkg:,customize:,extra:,push,proxy:,user:,registry:,clean,attempts:,update-id: -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +eval set -- "${OPTS}" + +while true; do + case $1 in + --) + # End of getopt arguments + shift + break + ;; + --version) + IMAGE_UPDATE_VER=$2 + shift 2 + ;; + --file) + read_params_from_file $2 + shift 2 + ;; + --from) + FROM=$2 + shift 2 + ;; + --wheel) + WHEELS+=($2) + shift 2 + ;; + --module-src) + MODULE_SRC+=($2) + shift 2 + ;; + --pkg) + DIST_PACKAGES+=($2) + shift 2 + ;; + --customize) + CUSTOMIZATION_SCRIPT=$2 + shift 2 + ;; + --extra) + EXTRA_FILES+=($2) + shift 2 + ;; + --push) + PUSH=yes + shift + ;; + --proxy) + PROXY=$2 + shift 2 + ;; + --user) + DOCKER_USER=$2 + shift 2 + ;; + --registry) + # Add a trailing / if needed + DOCKER_REGISTRY="${2%/}/" + shift 2 + ;; + --clean) + CLEAN=yes + shift + ;; + --attempts) + MAX_ATTEMPTS=$2 + shift 2 + ;; + --update-id) + UPDATE_ID=$2 + shift 2 + ;; + -h | --help ) + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac +done + +UPDATE_DIR=${MY_WORKSPACE}/std/update-images/${UPDATE_ID} + + +if [ -z "${FROM}" ]; then + echo "Image must be specified with --from option." >&2 + exit 1 +fi + +LABEL=$(basename ${FROM} | sed 's/:.*//') + +# Update the image version +CUR_IMAGE_UPDATE_VER=$(echo "${FROM}" | sed -r 's/.*\.([0-9][0-9]*)$/\1/') +if [ -z "${IMAGE_UPDATE_VER}" -o ${IMAGE_UPDATE_VER} = 0 ]; then + # IMAGE_UPDATE_VER is not set, so increment the current version + IMAGE_UPDATE_VER=$((${CUR_IMAGE_UPDATE_VER}+1)) +fi + +# Determine new tag for updated image +if [ -z "${CUR_IMAGE_UPDATE_VER}" ]; then + # The original image doesn't have a .VER at the end of the tag, + # so append the original tag with .IMAGE_UPDATE_VER + UPDATED_IMAGE="${FROM}.${IMAGE_UPDATE_VER}" +else + # Replace the .VER in the original image tag with .IMAGE_UPDATE_VER + UPDATED_IMAGE=$(echo ${FROM} | sed "s/\.[0-9][0-9]*$/\.${IMAGE_UPDATE_VER}/") +fi + +UPDATED_IMAGE_TAG=$(echo "${UPDATED_IMAGE}" | sed 's/.*://') + +# If DOCKER_USER and DOCKER_REGISTRY are specified, modify the UPDATED_IMAGE accordingly +if [ -n "${DOCKER_REGISTRY}" -o -n "${DOCKER_USER}" ]; then + UPDATED_IMAGE="${DOCKER_REGISTRY}${DOCKER_USER:-${USER}}/${LABEL}:${UPDATED_IMAGE_TAG}" +fi + +# Prepare the workspace for internal-update-stx-image.sh. +# The workspace will contain all files needed to install updates, +# structured in pip-packages, dist-packages, and extras directories +# as appropriate. + +WORKDIR=${UPDATE_DIR}/$(basename ${UPDATED_IMAGE} | tr ':' '_') +if [ -e ${WORKDIR} ]; then + rm -rf ${WORKDIR} +fi + +mkdir -p ${WORKDIR} +if [ $? -ne 0 ]; then + echo "Failed to create workdir: ${WORKDIR}" >&2 + exit 1 +fi + +# Change dir in case relative file locations were used +pushd ${FILE_BASEDIR} + +if [ -n "${CUSTOMIZATION_SCRIPT}" ]; then + if [ ! -f "${CUSTOMIZATION_SCRIPT}" ]; then + echo "Customization script not found: ${CUSTOMIZATION_SCRIPT}" >&2 + exit 1 + fi + + # Copy the customization script + cp ${CUSTOMIZATION_SCRIPT} ${WORKDIR}/customize.sh +fi + +copy_files_to_workdir ${WORKDIR}/extras ${EXTRA_FILES[@]} +copy_files_to_workdir ${WORKDIR}/pip-packages/wheels ${WHEELS[@]} +copy_files_to_workdir ${WORKDIR}/dist-packages ${DIST_PACKAGES[@]} + +if [ ${#MODULE_SRC[@]} -gt 0 ]; then + MODULES_DIR=${WORKDIR}/pip-packages/modules + mkdir -p ${MODULES_DIR} + if [ $? -ne 0 ]; then + echo "Failed to create dir: ${MODULES_DIR}" >&2 + exit 1 + fi + + for module_src in ${MODULE_SRC[@]}; do + src_location=$(echo "${module_src}" | awk -F'|' '{print $1}') + + if [ -d "${src_location}" ]; then + # Module source is a directory, so copy it to the workspace + cp --recursive --dereference ${src_location} ${MODULES_DIR} + if [ $? -ne 0 ]; then + echo "Failed to copy dir: ${src_location}" >&2 + exit 1 + fi + + module=$(basename ${src_location}) + module_ver=$(echo "${module_src}" | awk -F'|' '{print $2}') + hardcode_python_module_version ${MODULES_DIR}/${module} ${module_ver} + elif [[ ${src_location} =~ ^(http|https|git): ]]; then + # Module source is a URL, so use git to clone it. + # For a git repo, the module_src is specified as: + # src_location|module_ref|module_ver + # where: + # src_location - the URL of the repo to be cloned + # module_ref - optional specification of branch or tag to be fetched + # module_ver - optional specification of version to hardcode + + pushd ${MODULES_DIR} + + git clone ${src_location} + if [ $? -ne 0 ]; then + echo "Failed to clone src: ${src_location}" >&2 + exit 1 + fi + popd + + module=$(basename ${src_location} | sed 's/\.git$//') + + if [ ! -d "${MODULES_DIR}/${module}" ]; then + echo "Module directory doesn't exist: ${MODULES_DIR}/${module}" >&2 + exit 1 + fi + + module_ref=$(echo "${module_src}" | awk -F'|' '{print $2}') + if [ -n "${module_ref}" ]; then + pushd ${MODULES_DIR}/${module} + + git fetch ${src_location} ${module_ref} + if [ $? -ne 0 ]; then + echo "Failed to fetch repo branch: ${module} ${module_ref}" >&2 + exit 1 + fi + + git checkout FETCH_HEAD + if [ $? -ne 0 ]; then + echo "Failed to checkout FETCH_HEAD: ${module} ${module_ref}" >&2 + exit 1 + fi + popd + fi + + module_ver=$(echo "${module_src}" | awk -F'|' '{print $3}') + hardcode_python_module_version ${MODULES_DIR}/${module} ${module_ver} + else + echo "Invalid module source reference: ${src_location}" >&2 + exit 1 + fi + done +fi + +popd + +# Finally, copy the internal-update-stx-image.sh script +cp ${MY_SCRIPT_DIR}/internal-update-stx-image.sh ${WORKDIR}/ + +# WORKDIR is setup, let's pull the image and update it + +# Pull the image, even if already present, to ensure it's up to date +with_retries ${MAX_ATTEMPTS} docker image pull ${FROM} +if [ $? -ne 0 ]; then + echo "Failed to pull image: ${FROM}" >&2 + exit 1 +fi + +# Get the OS NAME from /etc/os-release +OS_NAME=$(docker run --rm ${FROM} bash -c 'source /etc/os-release && echo ${NAME}') + +# Run a container to install updates +UPDATE_CONTAINER=${USER}_${LABEL}_updater_$$ +docker run --name ${UPDATE_CONTAINER} \ + -v "${WORKDIR}":/image-update \ + ${FROM} \ + bash -x -c ' bash -x /image-update/internal-update-stx-image.sh ' +if [ $? -ne 0 ]; then + echo "Failed to update image: ${FROM}" >&2 + exit 1 +fi + +# Commit the updated image +docker commit --change='CMD ["bash"]' ${UPDATE_CONTAINER} ${UPDATED_IMAGE} +if [ $? -ne 0 ]; then + echo "Failed to commit updated image: ${UPDATE_CONTAINER}" >&2 + docker rm ${UPDATE_CONTAINER} >/dev/null + exit 1 +fi + +# Remove the update container +docker rm ${UPDATE_CONTAINER} >/dev/null + +if [ "${OS_NAME}" = "CentOS Linux" ]; then + # Record python modules and packages + docker run --rm ${UPDATED_IMAGE} bash -c 'rpm -qa' \ + | sort > ${UPDATE_DIR}/${LABEL}-${UPDATED_IMAGE_TAG}.rpmlst + if [ ${PIPESTATUS[0]} -ne 0 ]; then + echo "Failed to query RPMs from: ${UPDATED_IMAGE}" >&2 + exit 1 + fi + + docker run --rm ${UPDATED_IMAGE} bash -c 'pip freeze 2>/dev/null' \ + | sort > ${UPDATE_DIR}/${LABEL}-${UPDATED_IMAGE_TAG}.piplst + if [ ${PIPESTATUS[0]} -ne 0 ]; then + echo "Failed to query python modules from: ${UPDATED_IMAGE}" >&2 + exit 1 + fi +fi + +IMAGE_RECORD_FILE=${UPDATE_DIR}/image-updates.lst +update_image_record ${LABEL} ${UPDATED_IMAGE} ${IMAGE_RECORD_FILE} + +if [ "${PUSH}" = "yes" ]; then + docker push ${UPDATED_IMAGE} +fi + +if [ "${CLEAN}" = "yes" ]; then + docker image rm ${FROM} ${UPDATED_IMAGE} + if [ $? -ne 0 ]; then + echo "Failed to clean images from docker: ${FROM} ${UPDATED_IMAGE}" >&2 + fi +fi + +echo "Updated image: ${UPDATED_IMAGE}" + +exit 0 + diff --git a/build-tools/build-wheels/docker/docker-build-wheel.sh b/build-tools/build-wheels/docker/docker-build-wheel.sh index 60281036..e47de9b6 100755 --- a/build-tools/build-wheels/docker/docker-build-wheel.sh +++ b/build-tools/build-wheels/docker/docker-build-wheel.sh @@ -165,9 +165,16 @@ function from_git { continue fi - git checkout $branch + git fetch $gitrepo $branch if [ $? -ne 0 ]; then - echo "Failure running: git checkout $branch" + echo "Failure running: git fetch $gitrepo $branch" + echo $wheelname >> $FAILED_LOG + continue + fi + + git checkout FETCH_HEAD + if [ $? -ne 0 ]; then + echo "Failure running: git checkout FETCH_HEAD" echo $wheelname >> $FAILED_LOG continue fi