starlingx/root
Code Issues Proposed changes

Provide a new build mode for containers

Browse Source 
Providing a new build target --containers.

Container build targets typically need network access to
build, and do not need to be built as part of a regular build.

Container build targets are defined by: centos_pkg_dirs_containers
Container build targets can utilize the std and rt repos for
their build requirements.
Container target enables the mock flag: rpmbuild_networking

This also includes a fix where build-pkgs --installer in parallel
mode was building in serial mode instead.

Story: 2003909
Task: 27632
Change-Id: I3727a15fbdadc345a104c2af3446aee845171309
Signed-off-by: Al Bailey <Al.Bailey@windriver.com>
This commit is contained in:
Al Bailey 2018-11-08 08:19:41 -06:00
parent 1615f28109
commit aa15df5887
8 changed files with 118 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build-tools/build-avoidance-utils.sh
View File

@ -773,6 +773,11 @@ build_avoidance () {
return 1
fi
if [ "$BUILD_TYPE" == "containers" ]; then
>&2 echo "build_avoidance: BUILD_TYPE==containers not supported"
return 1
fi
build_avoidance_pre_clean $BUILD_TYPE
if [ $? -ne 0 ]; then
>&2 echo "Error: $FUNCNAME (${LINENO}): build_avoidance_pre_clean $BUILD_TYPE"

54
build-tools/build-pkgs-parallel
View File

@ -56,12 +56,13 @@ BUILD_AVOIDANCE_FLAG=0
STD_BUILD=1
RT_BUILD=1
INSTALLER_BUILD=0
CONTAINERS_BUILD=0
export BUILD_AVOIDANCE_URL=""
# read the options
TEMP=$(getopt -o h --long parallel,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@")
TEMP=$(getopt -o h --long parallel,rt,std,installer,containers,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-parallel' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -100,6 +101,7 @@ while true ; do
--rt) STD_BUILD=0 ; shift ;;
--std) RT_BUILD=0 ; shift ;;
--installer) INSTALLER_BUILD=1 ; STD_BUILD=0 ; RT_BUILD=0 ; shift ;;
--containers) INSTALLER_BUILD=0 ; STD_BUILD=0 ; RT_BUILD=0 ; CONTAINERS_BUILD=1 ; shift ;;
--parallel) shift ;;
--) shift ; break ;;
*) usage; exit 1 ;;
@ -166,6 +168,7 @@ EMPTY_TARGETS=" "
TARGETS_STD="$EMPTY_TARGETS"
TARGETS_RT="$EMPTY_TARGETS"
TARGETS_INSTALLER="$EMPTY_TARGETS"
TARGETS_CONTAINERS="$EMPTY_TARGETS"
TARGETS_MISC="$EMPTY_TARGETS"
find_targets () {
@ -261,11 +264,14 @@ if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_RT="$(find_targets centos_pkg_dirs_rt)"
BUILD_TYPE="installer"
TARGETS_INSTALLER="$(find_targets centos_pkg_dirs_installer)"
BUILD_TYPE="containers"
TARGETS_CONTAINERS="$(find_targets centos_pkg_dirs_containers)"
BUILD_TYPE="$BUILD_TYPE_SAVE"
echo "TARGETS_STD=$TARGETS_STD"
echo "TARGETS_RT=$TARGETS_RT"
echo "TARGETS_INSTALLER=$TARGETS_INSTALLER"
echo "TARGETS_CONTAINERS=$TARGETS_CONTAINERS"
for x in $TARGETS; do
if [[ $TARGETS_STD == *" $x "* ]]
@ -281,7 +287,13 @@ if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
echo "found $x" >> /dev/null;
INSTALLER_BUILD=1
else
TARGETS_MISC+="$x "
if [[ $TARGETS_CONTAINERS == *" $x "* ]]
then
echo "found $x" >> /dev/null;
CONTAINERS_BUILD=1
else
TARGETS_MISC+="$x "
fi
fi
fi
fi
@ -295,6 +307,7 @@ echo "TARGETS='$TARGETS'"
echo "TARGETS_STD='$TARGETS_STD'"
echo "TARGETS_RT='$TARGETS_RT'"
echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'"
echo "TARGETS_CONTAINERS='$TARGETS_CONTAINERS'"
echo "TARGETS_MISC='$TARGETS_MISC'"
if [ $CLEAN_FLAG -eq 1 ]; then
@ -333,13 +346,26 @@ if [ $CLEAN_FLAG -eq 1 ]; then
if [ $INSTALLER_BUILD -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_INSTALLER || exit 1
fi
fi
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_INSTALLER" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-serial --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --installer --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_INSTALLER || exit 1
fi
fi
if [ $CONTAINERS_BUILD -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_CONTAINERS"
${BUILD_PKGS_PARALLEL_DIR}/build-rpms-parallel --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_CONTAINERS || exit 1
fi
fi
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_CONTAINERS"
${BUILD_PKGS_PARALLEL_DIR}/build-srpms-parallel --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_CONTAINERS || exit 1
fi
fi
@ -364,7 +390,11 @@ function launch_build()
if [ "$build_type" == "installer" ]; then
targets="$TARGETS_INSTALLER $TARGETS_MISC"
else
targets="$TARGETS"
if [ "$build_type" == "containers" ]; then
targets="$TARGETS_CONTAINERS $TARGETS_MISC"
else
targets="$TARGETS"
fi
fi
fi
fi
@ -463,6 +493,16 @@ else
echo "Skipping 'installer' build"
fi
if [ $CONTAINERS_BUILD -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS" != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build containers
else
echo "Skipping 'containers' build, no valid targets in list: $TARGETS"
fi
else
echo "Skipping 'containers' build"
fi
# Make sure REFERENCE_BUILD is set to something
if [ -z $REFERENCE_BUILD ]; then
REFERENCE_BUILD=0

46
build-tools/build-pkgs-serial
View File

@ -56,12 +56,13 @@ BUILD_AVOIDANCE_FLAG=0
STD_BUILD=1
RT_BUILD=1
INSTALLER_BUILD=0
CONTAINERS_BUILD=0
export BUILD_AVOIDANCE_URL=""
# read the options
TEMP=$(getopt -o h --long serial,rt,std,installer,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@")
TEMP=$(getopt -o h --long serial,rt,std,installer,containers,edit,build-avoidance,no-build-avoidance,build-avoidance-dir:,build-avoidance-host:,build-avoidance-user:,build-avoidance-day:,no-meta-patch,no-descendants,no-required,no-build-info,no-autoclean,formal,careful,help,clean,append-log -n 'build-pkgs-serial' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -100,6 +101,7 @@ while true ; do
--rt) STD_BUILD=0 ; shift ;;
--std) RT_BUILD=0 ; shift ;;
--installer) INSTALLER_BUILD=1 ; STD_BUILD=0 ; RT_BUILD=0 ; shift ;;
--containers) INSTALLER_BUILD=0 ; STD_BUILD=0 ; RT_BUILD=0 ; CONTAINERS_BUILD=1;shift ;;
--serial) shift ;;
--) shift ; break ;;
*) usage; exit 1 ;;
@ -166,6 +168,7 @@ EMPTY_TARGETS=" "
TARGETS_STD="$EMPTY_TARGETS"
TARGETS_RT="$EMPTY_TARGETS"
TARGETS_INSTALLER="$EMPTY_TARGETS"
TARGETS_CONTAINERS="$EMPTY_TARGETS"
TARGETS_MISC="$EMPTY_TARGETS"
find_targets () {
@ -261,11 +264,14 @@ if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
TARGETS_RT="$(find_targets centos_pkg_dirs_rt)"
BUILD_TYPE="installer"
TARGETS_INSTALLER="$(find_targets centos_pkg_dirs_installer)"
BUILD_TYPE="containers"
TARGETS_CONTAINERS="$(find_targets centos_pkg_dirs_containers)"
BUILD_TYPE="$BUILD_TYPE_SAVE"
echo "TARGETS_STD=$TARGETS_STD"
echo "TARGETS_RT=$TARGETS_RT"
echo "TARGETS_INSTALLER=$TARGETS_INSTALLER"
echo "TARGETS_CONTAINERS=$TARGETS_CONTAINERS"
for x in $TARGETS; do
if [[ $TARGETS_STD == *" $x "* ]]
@ -281,7 +287,13 @@ if [ "$TARGETS" != "$EMPTY_TARGETS" ]; then
echo "found $x" >> /dev/null;
INSTALLER_BUILD=1
else
TARGETS_MISC+="$x "
if [[ $TARGETS_CONTAINERS == *" $x "* ]]
then
echo "found $x" >> /dev/null;
CONTAINERS_BUILD=1
else
TARGETS_MISC+="$x "
fi
fi
fi
fi
@ -295,6 +307,7 @@ echo "TARGETS='$TARGETS'"
echo "TARGETS_STD='$TARGETS_STD'"
echo "TARGETS_RT='$TARGETS_RT'"
echo "TARGETS_INSTALLER='$TARGETS_INSTALLER'"
echo "TARGETS_CONTAINERS='$TARGETS_CONTAINERS'"
echo "TARGETS_MISC='$TARGETS_MISC'"
if [ $CLEAN_FLAG -eq 1 ]; then
@ -343,6 +356,19 @@ if [ $CLEAN_FLAG -eq 1 ]; then
fi
fi
if [ $CONTAINERS_BUILD -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS" != "$EMPTY_TARGETS" ]; then
if [ $EDIT_FLAG -ne 1 ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_CONTAINERS"
${BUILD_PKGS_SERIAL_DIR}/build-rpms-serial --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_RPM $TARGETS_CONTAINERS || exit 1
fi
fi
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS" != "$EMPTY_TARGETS" ]; then
echo "${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_CONTAINERS"
${BUILD_PKGS_SERIAL_DIR}/build-srpms-serial --containers --clean $EXTRA_ARGS_COMMON $EXTRA_ARGS_SRPM $TARGETS_CONTAINERS || exit 1
fi
fi
exit $?
fi
@ -364,7 +390,11 @@ function launch_build()
if [ "$build_type" == "installer" ]; then
targets="$TARGETS_INSTALLER $TARGETS_MISC"
else
targets="$TARGETS"
if [ "$build_type" == "containers" ]; then
targets="$TARGETS_CONTAINERS $TARGETS_MISC"
else
targets="$TARGETS"
fi
fi
fi
fi
@ -462,6 +492,16 @@ else
echo "Skipping 'installer' build"
fi
if [ $CONTAINERS_BUILD -eq 1 ]; then
if [ "$TARGETS" == "$EMPTY_TARGETS" ] || [ "$TARGETS_CONTAINERS " != "$EMPTY_TARGETS" ] || [ "$TARGETS_MISC" != "$EMPTY_TARGETS" ]; then
launch_build containers
else
echo "Skipping 'containers' build, no valid targets in list: $TARGETS"
fi
else
echo "Skipping 'containers' build"
fi
# Make sure REFERENCE_BUILD is set to something
if [ -z $REFERENCE_BUILD ]; then
REFERENCE_BUILD=0

3
build-tools/build-rpms-parallel
View File

@ -1406,7 +1406,7 @@ FORMAL_FLAG=0
CAREFUL=0
# read the options
TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,tmpfs-clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
@ -1436,6 +1436,7 @@ while true ; do
--std) BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
-h|--help) HELP=1 ; shift ;;
--clean) CLEAN_FLAG=1 ; shift ;;
--tmpfs-clean) if [ -n "$MY_WORKSPACE" ]; then export MY_WORKSPACE=$MY_WORKSPACE/$BUILD_TYPE; exit 0; fi ;;

3
build-tools/build-rpms-serial
View File

@ -1175,7 +1175,7 @@ FORMAL_FLAG=0
CAREFUL=0
# read the options
TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,no-required,no-descendants,no-autoclean,no-build-info,clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,containers,no-required,no-descendants,no-autoclean,no-build-info,clean,formal,careful,help,arga::,argb,argc: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
@ -1205,6 +1205,7 @@ while true ; do
--std) BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
-h|--help) HELP=1 ; shift ;;
--clean) CLEAN_FLAG=1 ; shift ;;
-c|--argc)

11
build-tools/build-srpms-parallel
View File

@ -74,19 +74,19 @@ usage () {
echo ""
echo "Usage: "
echo " Create source rpms:"
echo " $ME [--rt | --std | --installer] [--no-descendants] [--formal] [ list of package names ]"
echo " $ME [--rt | --std | --installer | --containers] [--no-descendants] [--formal] [ list of package names ]"
echo ""
echo " Delete source rpms, and the directories associated with it's creation:"
echo " Note: does not clean an edit environment"
echo " $ME --clean [--rt | --std | --installer] [optional list of package names]"
echo " $ME --clean [--rt | --std | --installer | --containers] [optional list of package names]"
echo ""
echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents,"
echo " one for source code and one for metadata such as the spec file."
echo " If --no-meta-patch is specified, then WRS patches are omitted."
echo " $ME --edit [--rt | --std | --installer] [--no-meta-patch] [list of package names]"
echo " $ME --edit [--rt | --std | --installer | --containers] [--no-meta-patch] [list of package names]"
echo ""
echo " Delete an edit environment"
echo " $ME --edit --clean [--rt | --std | --installer] [list of package names]"
echo " $ME --edit --clean [--rt | --std | --installer | --containers] [list of package names]"
echo ""
echo " This help page"
echo " $ME --help"
@ -160,7 +160,7 @@ EDIT_FLAG=0
NO_META_PATCH_FLAG=0
# read the options
TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
TEMP=$(getopt -o ha::bc: --long parallel,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
@ -193,6 +193,7 @@ while true ; do
--std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE_FLAG=1; BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
--edit) EDIT_FLAG=1 ; shift ;;
--no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;;
--parallel) shift ;;

11
build-tools/build-srpms-serial
View File

@ -61,19 +61,19 @@ usage () {
echo ""
echo "Usage: "
echo " Create source rpms:"
echo " $ME [--rt | --std | --installer] [--no-descendants] [--formal] [ list of package names ]"
echo " $ME [--rt | --std | --installer | --containers] [--no-descendants] [--formal] [ list of package names ]"
echo ""
echo " Delete source rpms, and the directories associated with it's creation:"
echo " Note: does not clean an edit environment"
echo " $ME --clean [--rt | --std | --installer] [optional list of package names]"
echo " $ME --clean [--rt | --std | --installer | --containers] [optional list of package names]"
echo ""
echo " Extract an src.rpm into a pair of git trees to aid in editing it's contents,"
echo " one for source code and one for metadata such as the spec file."
echo " If --no-meta-patch is specified, then WRS patches are omitted."
echo " $ME --edit [--rt | --std | --installer] [--no-meta-patch] [list of package names]"
echo " $ME --edit [--rt | --std | --installer | --containers] [--no-meta-patch] [list of package names]"
echo ""
echo " Delete an edit environment"
echo " $ME --edit --clean [--rt | --std | --installer] [list of package names]"
echo " $ME --edit --clean [--rt | --std | --installer | --containers] [list of package names]"
echo ""
echo " This help page"
echo " $ME --help"
@ -146,7 +146,7 @@ EDIT_FLAG=0
NO_META_PATCH_FLAG=0
# read the options
TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
TEMP=$(getopt -o ha::bc: --long serial,std,rt,installer,containers,no-descendants,no-meta-patch,no-build-info,help,formal,clean,edit,arga::,argb,argc: -n "$ME" -- "$@")
if [ $? -ne 0 ]; then
usage
@ -179,6 +179,7 @@ while true ; do
--std) BUILD_TYPE_FLAG=1; BUILD_TYPE=std; shift ;;
--rt) BUILD_TYPE_FLAG=1; BUILD_TYPE=rt; shift ;;
--installer) BUILD_TYPE=installer; shift ;;
--containers) BUILD_TYPE=containers; shift ;;
--edit) EDIT_FLAG=1 ; shift ;;
--no-meta-patch) NO_META_PATCH_FLAG=1 ; shift ;;
--serial) shift ;;

8
build-tools/modify-build-cfg
View File

@ -95,9 +95,15 @@ fi
grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum'" $FILE || \
echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum'" >> $FILE
# Special case for containers.
# rpmbuild_networking is required for invoking helm commands within mock
grep -q "config_opts\['rpmbuild_networking'\] = True" $FILE || \
# building containers requires the std repo to be enabled.
if [ "containers" == "$BUILD_TYPE" ]; then
grep -q "config_opts\['rpmbuild_networking'\] = True" $FILE || \
echo "config_opts['rpmbuild_networking'] = True" >> $FILE
sed -i "/^\[local-std\]/,/^\[/ s/enabled=0/enabled=1/" $FILE
fi
#
# Read macros from tis.macros to add to the build config file,
# for use in RPM spec files