Enable build of docker images, charts and wheels from a layered build

1) create merged wheels.inc file(s) as part of build-pkgs.
These files can be published by CENGN layer builds, and
be referenced by the 'container' layer build.

2) get-stx-wheels.sh needs to find wheels.inc files from
lower layer builds.  The will be fould under
${MY_REPO}/cgcs-centos-repo/layer_wheels_inc.

3) build-helm-charts.sh must now also search for helm
rpms under ${MY_REPO}/cgcs-centos-repo/Binary/noarch
for chart packages built by lower layers.

Story: 2006166
Task: 38979
Depends-On: https://review.opendev.org/711773
Depends-On: https://review.opendev.org/711774
Change-Id: I91650d3e1746a45c8ac118cca58e9b64f30c5b72
Signed-off-by: Scott Little <scott.little@windriver.com>

build-tools/build-helm-charts.sh

@@ -149,6 +149,16 @@ function build_image_versions_to_manifest {
     done
 }
 
+function find_chartfile {
+    local helm_rpm=$1
+
+    for RPMS_DIR in ${RPMS_DIRS}; do
+        if [ -d ${RPMS_DIR} ]; then
+            find ${RPMS_DIR} -name "${helm_rpm}${FIND_GLOB}"
+        fi
+    done
+}
+
 # Extract the helm charts from a rpm
 function extract_chartfile {
     local helm_rpm=$1
@@ -157,8 +167,7 @@ function extract_chartfile {
         centos)
             # Bash globbing does not handle [^-] like regex
             # so grep needed to be used
-            rpm_file=$(ls ${RPMS_DIR} | grep "^${helm_rpm}${GREP_GLOB}")
+            chartfile=$(find_chartfile ${helm_rpm})
-            chartfile=${RPMS_DIR}/${rpm_file}
             if [ ! -f ${chartfile} ]; then
                 echo "Failed to find helm package: ${helm_rpm}" >&2
                 exit 1
@@ -422,8 +431,9 @@ if [ ${#IMAGE_RECORDS[@]} -ne 0 ]; then
 fi
 
 # Extract helm charts and app version from the application rpm
-RPMS_DIR=${MY_WORKSPACE}/std/rpmbuild/RPMS
+RPMS_DIRS="${MY_WORKSPACE}/std/rpmbuild/RPMS ${MY_REPO}/cgcs-centos-repo/Binary/noarch"
-GREP_GLOB="-[^-]*-[^-]*.tis.noarch.rpm"
+FIND_GLOB="*.tis.noarch.rpm"
+
 extract_application_rpms
 # Extract helm charts from the application dependent rpms
 if [ ${#APP_HELM_FILES[@]} -gt 0 ]; then

build-tools/build-rpms-parallel

@@ -88,6 +88,7 @@ export MOCK=/usr/bin/mock
 
 BUILD_RPMS_PARALLEL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 source "${BUILD_RPMS_PARALLEL_DIR}/image-utils.sh"
+source "${BUILD_RPMS_PARALLEL_DIR}/wheel-utils.sh"
 source "${BUILD_RPMS_PARALLEL_DIR}/spec-utils"
 source "${BUILD_RPMS_PARALLEL_DIR}/srpm-utils"
 
@@ -1485,6 +1486,11 @@ image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}"
 DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc"
 image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}"
 
+for STREAM in stable dev; do
+    WHEELS_INC_FILE="${MY_WORKSPACE}/${DISTRO}_${STREAM}_wheels.inc"
+    wheels_inc_list ${STREAM} ${DISTRO} > "${WHEELS_INC_FILE}"
+done
+
 LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
 
 TARGETS=$@

build-tools/build-rpms-serial

@@ -51,6 +51,7 @@ export MOCK=/usr/bin/mock
 
 BUILD_RPMS_SERIAL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 source "${BUILD_RPMS_SERIAL_DIR}/image-utils.sh"
+source "${BUILD_RPMS_SERIAL_DIR}/wheel-utils.sh"
 source "${BUILD_RPMS_SERIAL_DIR}/spec-utils"
 source "${BUILD_RPMS_SERIAL_DIR}/srpm-utils"
 
@@ -1253,6 +1254,11 @@ image_inc_list iso std ${DISTRO} > "${IMAGE_INC_FILE}"
 DEV_IMAGE_INC_FILE="${MY_WORKSPACE}/image-dev.inc"
 image_inc_list iso dev ${DISTRO} > "${DEV_IMAGE_INC_FILE}"
 
+for STREAM in stable dev; do
+    WHEELS_INC_FILE="${MY_WORKSPACE}/${DISTRO}_${STREAM}_wheels.inc"
+    wheels_inc_list ${STREAM} ${DISTRO} > "${WHEELS_INC_FILE}"
+done
+
 LAST_PLATFORM_RELEASE_FILE="$MY_BUILD_DIR/.platform_release"
 
 TARGETS=$@

build-tools/build-wheels/get-stx-wheels.sh

@@ -88,7 +88,23 @@ function get_wheels_files {
     find ${GIT_LIST} -maxdepth 1 -name "${OS}_${BUILD_STREAM}_wheels.inc"
 }
 
-declare -a WHEELS_FILES=($(get_wheels_files))
+function get_lower_layer_wheels_files {
+    find ${MY_REPO}/cgcs-centos-repo/layer_wheels_inc -maxdepth 1 -name "*_${OS}_${BUILD_STREAM}_wheels.inc"
+}
+
+function find_wheel_rpm {
+    local wheel="$1"
+    local repo=
+
+    for repo in ${MY_WORKSPACE}/std/rpmbuild/RPMS \
+                ${MY_REPO}/cgcs-centos-repo/Binary; do
+        if [ -d $repo ]; then
+            find $repo -name "${wheel}-[^-]*-[^-]*[.][^.]*[.]rpm"
+        fi
+    done | head -n 1
+}
+
+declare -a WHEELS_FILES=($(get_wheels_files) $(get_lower_layer_wheels_files))
 if [ ${#WHEELS_FILES[@]} -eq 0 ]; then
     echo "Could not find ${OS} wheels.inc files" >&2
     exit 1
@@ -109,9 +125,9 @@ for wheel in $(sed -e 's/#.*//' ${WHEELS_FILES[@]} | sort -u); do
         centos)
             # Bash globbing does not handle [^\-] well,
             # so use grep instead
-            wheelfile=$(ls ${MY_WORKSPACE}/std/rpmbuild/RPMS/${wheel}-* | grep -- '[^\-]*-[^\-]*.rpm')
+            wheelfile="$(find_wheel_rpm ${wheel})"
 
-            if [ ! -f "${wheelfile}" ]; then
+            if [ ! -e "${wheelfile}" ]; then
                 echo "Could not find ${wheel}" >&2
                 FAILED+=($wheel)
                 continue

build-tools/wheel-utils.sh

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# A place for any functions related to wheels.inc files
+#
+
+WHEEL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+
+source "${WHEEL_UTILS_DIR}/git-utils.sh"
+
+#
+# wheels_inc_list <stream> <distro> [<layer>]
+#
+# Parameters:
+#    stream:    One of 'stable', 'dev'
+#    distro:    One of 'centos', ...
+#
+# Returns: A list of unique rpm packages that contain needed wheel
+#          files.  This is the union per git wheels.inc files.
+
+wheels_inc_list () {
+    local stream=$1
+    local distro=$2
+
+    local search_target=${distro}_${stream}_wheels.inc
+
+    (
+    for d in $GIT_LIST; do
+        find $d/ -maxdepth 1 -name "${search_target}" -exec grep '^[^#]' {} +
+    done
+    ) | sort --unique
+}