Merge "Tool changes to allow image.inc to be split across git repos."

build-tools/build-guest

@@ -1,10 +1,17 @@
 #!/bin/env bash
 
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
 #
 # Build the tis-centos-image.img or tis-centos-image-rt.img file
 #
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BUILD_GUEST_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+source "${BUILD_GUEST_DIR}/image-utils.sh"
 
 # NOTE: TMP_DIR must end in '/'
 # NOTE: /tmp/ is now tmpfs like.  Can't be trusted across multiple mock commands
@@ -35,7 +42,7 @@ function init_vars {
    fi
 
    # Path to guest configuration
-   GUEST_BUILD_DIR=$DIR/build_guest
+   GUEST_BUILD_DIR="${BUILD_GUEST_DIR}/build_guest"
    GUEST_BUILD_CMD=$GUEST_BUILD_DIR/build-guest-image.py
    if [ $VERBOSE -eq 1 ]; then
        GUEST_BUILD_CMD="$GUEST_BUILD_CMD -x"
@@ -108,7 +115,7 @@ function create_rootfs {
     GUEST_ENV="${MY_BUILD_ENVIRONMENT}-guest"
     GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE
 
-    MY_BUILD_ENVIRONMENT=$GUEST_ENV ${DIR}/modify-build-cfg $GUEST_CFG
+    MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG
     if [ $? -ne 0 ]; then
 	printf "   Error -- Could not update $GUEST_CFG\n";
 	exit 1
@@ -144,10 +151,10 @@ function create_rootfs {
 
     if [ $BUILD_MODE == 'std' ]; then
        INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list.txt)
-       TIS_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/image.inc)
+       TIS_RPM_LIST=$(image_inc_list guest std centos)
     elif [ $BUILD_MODE == 'rt' ]; then
        INC_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/rpm-install-list-rt.txt)
-       TIS_RPM_LIST=$(grep -v '^#' ${GUEST_BUILD_DIR}/image-rt.inc)
+       TIS_RPM_LIST=$(image_inc_list guest rt centos)
        EXTRA_REPOS="--enablerepo local-rt"
     else
        printf "   Error -- unknown BUILD_MODE '$BUILD_MODE'\n";
@@ -156,6 +163,9 @@ function create_rootfs {
 
     $MOCK -r $GUEST_CFG --install ${INC_RPM_LIST} ${TIS_RPM_LIST} ${EXTRA_REPOS}
     if [ $? -ne 0 ]; then
+        printf "=====\n"
+        cat $GUEST_DIR/mock/result/root.log | sed -n '/Error:/,$p' | sed '/Child return code was:/q'
+        printf "=====\n"
 	printf "   Error -- Failed to install RPM packages\n";
 	exit 1
     fi
@@ -260,7 +270,7 @@ function clean_guest {
     GUEST_CFG=$GUEST_DIR/$MY_BUILD_ENVIRONMENT_FILE
 
     if [ ! -e $GUEST_CFG ]; then
-	MY_BUILD_ENVIRONMENT=$GUEST_ENV ${DIR}/modify-build-cfg $GUEST_CFG
+	MY_BUILD_ENVIRONMENT=$GUEST_ENV "${BUILD_GUEST_DIR}/modify-build-cfg" $GUEST_CFG
 	if [ $? -ne 0 ]; then
 	    printf "   Error -- Could not update $GUEST_CFG\n";
 	    exit 1

build-tools/build-iso

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
 # Build the export/bootimage.iso file
 #
 # This script uses environment variables to determine the source of
@@ -8,6 +14,9 @@
 # It starts by building a basic "vanilla CentOS" ISO, and then adds our
 # packages to it.
 
+BUILD_ISO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+source "${BUILD_ISO_DIR}/image-utils.sh"
+
 usage () {
     echo ""
     echo "Usage: "
@@ -274,6 +283,10 @@ function init_output_dir {
    # This directory will contains files required for the PXE network installer
    \mkdir -p $OUTPUT_DIST_DIR/isolinux/pxeboot
 
+   echo "  Preparing package lists"
+   image_inc_list iso std centos > "${PKGLIST_TI}"
+   image_inc_list iso dev centos > "${PKGLIST_DEV}"
+
    echo "  Copying base files"
 
    # Generate .discinfo file
@@ -664,27 +677,27 @@ printf   "*************************\n\n"
 # Init variables
 init_vars
 check_vars
-PKGLIST_MINIMAL=$INTERNAL_REPO_ROOT/build-tools/build_iso/minimal_rpm_list.txt
+PKGLIST_MINIMAL="${INTERNAL_REPO_ROOT}/build-tools/build_iso/minimal_rpm_list.txt"
-PKGLIST_TI=$INTERNAL_REPO_ROOT/build-tools/build_iso/image.inc
+PKGLIST_TI="${OUTPUT_DIR}/image.inc"
-PKGLIST_DEV=$INTERNAL_REPO_ROOT/build-tools/build_iso/image-dev.inc
+PKGLIST_DEV="${OUTPUT_DIR}/image-dev.inc"
 
 # Create skeleton build dir
 init_output_dir
 
 # Create the vanilla DVD
 echo "Copying vanilla CentOS RPMs"
-install_pkg_list $PKGLIST_MINIMAL
+install_pkg_list "${PKGLIST_MINIMAL}"
 
 # Find all CGCS packages
 # SAL exit 0
 echo "Installing Titanium Cloud packages"
-install_pkg_list $PKGLIST_TI
+install_pkg_list "${PKGLIST_TI}"
 if [ $? -eq 2 ]; then
 	exit 1
 fi
 if [ "x${RELEASE_BUILD}" == "x" ]; then
     echo "Installing Titanium Cloud developer packages"
-    install_pkg_list ${PKGLIST_DEV}
+    install_pkg_list "${PKGLIST_DEV}"
     if [ $? -eq 2 ]; then
         exit 1
     fi
@@ -711,7 +724,7 @@ final_touches
 # Sign the ISO
 sign_iso
 
-make_report $PKGLIST_MINIMAL $PKGLIST_TI
+make_report "${PKGLIST_MINIMAL}" "${PKGLIST_TI}"
 
 # Check sanity
 FILESIZE=$(wc -c <"$OUTPUT_FILE")

build-tools/build-rpms-parallel

@@ -1,10 +1,20 @@
 #!/bin/bash
-# set -x
+
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
 
 export ME=$(basename "$0")
 CMDLINE="$ME $@"
 
 
+
+# Build for distribution.  Currently 'centos' is only supported value.
+export DISTRO="centos"
+
 # Maximum number of parallel build environments
 ABSOLUTE_MAX_WORKERS=4
 
@@ -52,16 +62,16 @@ RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm"
 SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm"
 
 UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns"
-IMAGE_INC_FILE="$MY_REPO/build-tools/build_iso/image.inc"
 
 SIGN_SECURE_BOOT="sign-secure-boot"
 SIGN_SECURE_BOOT_LOG="sign-secure-boot.log"
 
 export MOCK=/usr/bin/mock
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
-source $DIR/spec-utils
+source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh"
-source $DIR/srpm-utils
+source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils"
+source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils"
 
 HOME=$(pwd)
 
@@ -1450,11 +1460,10 @@ export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg
 export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
 export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
 
-LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
+IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
+image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}"
 
-if [ "$BUILD_TYPE" != "std" ]; then
+LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
-   PKG_DIRS_FILE=centos_pkg_dirs_$BUILD_TYPE
-fi
 
 TARGETS=$@
 
@@ -1548,7 +1557,7 @@ export BUILD_CFG="$MY_BUILD_CFG"
 # Place build-time environement variables in mock environment
 echo "FORMAL_BUILD=$FORMAL_BUILD"
 echo "modify-build-cfg $BUILD_CFG"
-${DIR}/modify-build-cfg $BUILD_CFG
+${BUILD_RPMS_PARALLEL_DIR}/modify-build-cfg $BUILD_CFG
 if [ $? -ne 0 ]; then
        echo "Could not modifiy $BUILD_CFG";
        exit 1

build-tools/build-rpms-serial

@@ -1,5 +1,10 @@
 #!/bin/bash
-# set -x
+
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
 
 export ME=$(basename "$0")
 CMDLINE="$ME $@"
@@ -21,16 +26,16 @@ RPM_TO_SRPM_MAP_FILE="$DEPENDANCY_DIR/rpm-to-srpm"
 SRPM_TO_RPM_MAP_FILE="$DEPENDANCY_DIR/srpm-to-rpm"
 
 UNBUILT_PATTERN_FILE="$MY_REPO/build-data/unbuilt_rpm_patterns"
-IMAGE_INC_FILE="$MY_REPO/build-tools/build_iso/image.inc"
 
 SIGN_SECURE_BOOT="sign-secure-boot"
 SIGN_SECURE_BOOT_LOG="sign-secure-boot.log"
 
 export MOCK=/usr/bin/mock
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BUILD_RPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
-source $DIR/spec-utils
+source "${BUILD_RPMS_SERIAL_DIR}/image-utils.sh"
-source $DIR/srpm-utils
+source "${BUILD_RPMS_SERIAL_DIR}/spec-utils"
+source "${BUILD_RPMS_SERIAL_DIR}/srpm-utils"
 
 HOME=$(pwd)
 
@@ -548,10 +553,8 @@ export MY_BUILD_ENVIRONMENT_FILE=$MY_BUILD_ENVIRONMENT.cfg
 export MY_BUILD_CFG=$MY_WORKSPACE/$MY_BUILD_ENVIRONMENT_FILE
 export MY_MOCK_ROOT=$MY_WORKSPACE/mock/root
 
-
+IMAGE_INC_FILE="${MY_WORKSPACE}/image.inc"
-if [ "$BUILD_TYPE" != "std" ]; then
+image_inc_list iso ${BUILD_TYPE} ${DISTRO} > "${IMAGE_INC_FILE}"
-   PKG_DIRS_FILE=centos_pkg_dirs_$BUILD_TYPE
-fi
 
 TARGETS=$@
 
@@ -645,7 +648,7 @@ export BUILD_CFG="$MY_BUILD_CFG"
 # Place build-time environement variables in mock environment
 echo "FORMAL_BUILD=$FORMAL_BUILD"
 echo "modify-build-cfg $BUILD_CFG"
-${DIR}/modify-build-cfg $BUILD_CFG
+${BUILD_RPMS_SERIAL_DIR}/modify-build-cfg $BUILD_CFG
 if [ $? -ne 0 ]; then
        echo "Could not modifiy $BUILD_CFG";
        exit 1

build-tools/git-utils.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# A place for any functions relating to git, or the git hierarchy created 
+# by repo manifests.
+#
+
+# GIT_LIST: A list of root directories for all the gits under $MY_REPO
+export GIT_LIST=$(find $MY_REPO -type d -name '.git' -exec dirname {} \;)
+

build-tools/image-utils.sh

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# A place for any functions related to image.inc files
+#
+
+IMAGE_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+
+source "${IMAGE_UTILS_DIR}/git-utils.sh"
+
+#
+# image_inc_list <build_target> <build_type> <distro>
+#
+# Parameters:
+#    build_target: One of 'iso', 'guest' ...
+#    build_type:   One of 'std', 'rt', 'dev' ...
+#    distro:       One of 'centos', ...
+#
+# Returns: A list of unique package that must be included for
+#          the desired distro's build target and build type.
+#          This is the union of the global and per git 
+#          image.inc files.
+
+image_inc_list () {
+    local build_target=$1
+    local build_type=$2
+    local distro=$3
+
+    local root_file=""
+    local build_type_extension=""
+    local search_target=""
+
+    if [ "${build_type}" != "std" ]; then
+        build_type_extension="_${build_type}"
+        build_type_extension_bt="-${build_type}"
+    fi
+
+    root_dir="${MY_REPO}/build-tools/build_${build_target}"
+    root_file="${root_dir}/image${build_type_extension_bt}.inc"
+    search_target=${distro}_${build_target}_image${build_type_extension}.inc
+
+    (
+    if [ -f ${root_file} ]; then
+        grep '^[^#]' ${root_file}
+    fi
+    for d in $GIT_LIST; do
+        find $d -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} +
+    done
+    ) | sort --unique
+}