From 67ec45580d7ced9ddf865e969d7230b95d7c0b36 Mon Sep 17 00:00:00 2001
From: Angie Wang <angie.wang@windriver.com>
Date: Mon, 15 Apr 2019 12:16:13 -0400
Subject: [PATCH] Build versioned application tarball with metadata

This commit updates the chart build to build a metadata file into
an application tarball. The metadata file includes the app name,
app version and patch dependencies if there are patches specified
via option --patch-dependencies. The app name can be passed via
option --app, the app version is constructed with the app rpm version
and label.

The name of application tarball is renamed to the app name plus
the app version.

Change-Id: I8ce54c3beac3d671f580fec6c2e4c37153e163f4
Depends-On: https://review.opendev.org/#/c/655789/
Story: 2005350
Task: 30420
Signed-off-by: Angie Wang <angie.wang@windriver.com>
---
 build-tools/build-helm-charts.sh | 378 ++++++++++++++++++++++---------
 1 file changed, 266 insertions(+), 112 deletions(-)

diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh
index 405f0ef4..fcf466c7 100755
--- a/build-tools/build-helm-charts.sh
+++ b/build-tools/build-helm-charts.sh
@@ -18,28 +18,182 @@ fi
 SUPPORTED_OS_ARGS=('centos')
 OS=centos
 LABEL=""
-declare -a IMAGE_FILES
+APP_NAME="stx-openstack"
+APP_VERSION=""
+declare -a IMAGE_RECORDS
+declare -a PATCH_DEPENDENCIES
 
 function usage {
     cat >&2 <<EOF
 Usage:
-$(basename $0) [ --os <os> ] [-i, --image-file <image-file>] [--label <label>] [ --verbose ]
+$(basename $0) [ --os <os> ] [-a, --app <app-name>] [-i, --image-record <image-record>] [--label <label>] [-p, --patch-dependency <patch-dependency>] [ --verbose ]
 Options:
-    --os:                 Specify base OS (eg. centos)
-    -i, --image-file:     Specify the path to image file(s) or url(s). Multiple
-                          files/urls can be specified with a comma-separated list,
-                          or with multiple --image-file arguments.
-                          Note: Files are in order of priority. Images may appear
-                          in multiple files, the last image reference has higher
-                          priority.
-    --label:              Specify the label of the application tarball. The label
-                          is used to construct the name of tarball.
-    --verbose:            Verbose output
-    --help:               Give this help list
+    --os:
+            Specify base OS (eg. centos)
+
+    -a, --app:
+            Specify the application name
+
+    -i, --image-record:
+            Specify the path to image record file(s) or url(s).
+            Multiple files/urls can be specified with a comma-separated
+            list, or with multiple --image-record arguments.
+            Note: Files are in order of priority. Images may appear
+            in multiple files, the last image reference has higher
+            priority.
+
+    -l, --label:
+            Specify the label of the application tarball. The label
+            is used to construct the name of tarball.
+
+    -p, --patch-dependency:
+            Specify the patch dependency of the application tarball.
+            Multiple patches can be specified with a comma-separated
+            list, or with multiple --patch-dependency arguments.
+
+    --verbose:
+            Verbose output
+
+    --help:
+            Give this help list
 EOF
 }
 
-OPTS=$(getopt -o h,i: -l help,os:,image-file:,label:,verbose -- "$@")
+function is_in {
+    local search=$1
+    shift
+
+    for v in $*; do
+        if [ "${search}" = "${v}" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Read the image versions from the passed image
+# record files and build them into armada manifest
+function build_image_versions_to_manifest {
+    local manifest_file=$1
+
+    for image_record in ${IMAGE_RECORDS[@]}; do
+
+        if [[ ${image_record} =~ ^https?://.*(.lst|.txt)$ ]]; then
+            wget --quiet --no-clobber ${image_record} \
+                 --directory-prefix ${IMAGE_RECORD_PATH}
+
+            if [ $? -ne 0 ]; then
+                echo "Failed to download image record file from ${image_record}" >&2
+                exit 1
+            fi
+        elif [[ -f ${image_record} && ${image_record} =~ .lst|.txt ]]; then
+            cp ${image_record} ${IMAGE_RECORD_PATH}
+            if [ $? -ne 0 ]; then
+                echo "Failed to copy ${image_record} to ${IMAGE_RECORD_PATH}" >&2
+                exit 1
+            fi
+        else
+            echo "Cannot recognize the provided image record file:${image_record}" >&2
+            exit 1
+        fi
+
+        # An image record file contains a list of images with the following format:
+        # <docker-registry>/<repository>/<repository>/.../<image-name>:<tag>
+        #
+        # An example of the content of an image record file:
+        # e.g. images-centos-dev-latest.lst
+        # docker.io/starlingx/stx-aodh:master-centos-dev-latest
+        # docker.io/starlingx/stx-ceilometer:master-centos-dev-latest
+        # docker.io/starlingx/stx-cinder:master-centos-dev-latest
+        # ...
+        #
+        # An example of the usage of an image reference in manifest file:
+        # e.g. manifest.yaml
+        # images:
+        #   tags:
+        #     aodh_api: docker.io/starlingx/stx-aodh:master-centos-stable-latest
+        #     aodh_db_sync: docker.io/starlingx/stx-aodh:master-centos-stable-latest
+        #     ...
+        #
+        # To replace the images in the manifest file with the images in image record file:
+        # For each image reference in the image record file,
+        # 1. extract image name
+        #    e.g. image_name = stx-aodh
+        #
+        # 2. search the image reference in manifest yaml via image_name
+        #    e.g. old_image_reference = docker.io/starlingx/stx-aodh:master-centos-stable-latest
+        #
+        # 3. update the manifest file to replace the old image references with the new one
+        #    e.g. manifest.yaml
+        #    images:
+        #      tags:
+        #        aodh_api: docker.io/starlingx/stx-aodh:master-centos-dev-latest
+        #        aodh_db_sync: docker.io/starlingx/stx-aodh:master-centos-dev-latest
+        #
+        image_record=${IMAGE_RECORD_PATH}/$(basename ${image_record})
+        for image_reference in $(cat ${image_record}); do
+            image_name=$(echo ${image_reference} | sed -n 's/.*\/\(.*\):.*$/\1/p')
+            old_image_reference="\([a-zA-Z0-9.]*\|[0-9.:]*\)\/.*${image_name}:.*"
+
+            sed -i "s#${old_image_reference}#${image_reference}#" ${manifest_file}
+            if [ $? -ne 0 ]; then
+                echo "Failed to update manifest file" >&2
+                exit 1
+            fi
+        done
+    done
+}
+
+# Extract the helm charts from a rpm
+function extract_chartfile {
+    local helm_rpm=$1
+
+    case $OS in
+        centos)
+            # Bash globbing does not handle [^-] like regex
+            # so grep needed to be used
+            rpm_file=$(ls ${RPMS_DIR} | grep "^${helm_rpm}${GREP_GLOB}")
+            chartfile=${RPMS_DIR}/${rpm_file}
+            if [ ! -f ${chartfile} ]; then
+                echo "Failed to find helm package: ${helm_rpm}" >&2
+                exit 1
+            else
+                rpm2cpio ${chartfile} | cpio ${CPIO_FLAGS}
+                if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then
+                    echo "Failed to extract content of helm package: ${chartfile}" >&2
+                    exit 1
+                fi
+            fi
+
+            ;;
+        *)
+            echo "Unsupported OS ${OS}" >&2
+            ;;
+    esac
+}
+
+# Extract the helm charts and information from the application rpm
+function extract_application_rpm {
+    extract_chartfile ${APP_RPM}
+
+    APP_VERSION=$(rpm -qp --qf '%{VERSION}-%{RELEASE}' ${chartfile} | sed 's/\.tis//')
+    if [ -z "${APP_VERSION}" ]; then
+        echo "Failed to get the application version" >&2
+        exit 1
+    fi
+
+    helm_files=$(rpm -qpR ${chartfile})
+    if [ $? -ne 0 ]; then
+        echo "Failed to get the helm rpm dependencies for ${APP_RPM}" >&2
+        exit 1
+    fi
+
+    # Get rid of the rpmlib dependencies
+    APP_HELM_FILES=($(echo ${helm_files} | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g'))
+}
+
+# TODO(awang): remove the deprecated image-file option
+OPTS=$(getopt -o h,a:,i:,l:,p: -l help,os:,app:,image-record:,image-file:,label:,patch-dependency:,verbose -- "$@")
 if [ $? -ne 0 ]; then
     usage
     exit 1
@@ -58,15 +212,24 @@ while true; do
             OS=$2
             shift 2
             ;;
-        -i | --image-file)
-            # Read comma-separated values into array
-            IMAGE_FILES+=(${2//,/ })
+        -a | --app)
+            APP_NAME=$2
             shift 2
             ;;
-        --label)
+        -i | --image-record | --image-file)
+            # Read comma-separated values into array
+            IMAGE_RECORDS+=(${2//,/ })
+            shift 2
+            ;;
+        -l | --label)
             LABEL=$2
             shift 2
             ;;
+        -p | --patch-dependency)
+            # Read comma-separated values into array
+            PATCH_DEPENDENCIES+=(${2//,/ })
+            shift 2
+            ;;
         --verbose)
             VERBOSE=true
             shift
@@ -103,110 +266,69 @@ if [ ${VALID_OS} -ne 0 ]; then
     exit 1
 fi
 
-source ${MY_REPO}/build-tools/git-utils.sh
-
-function get_helm_files {
-    find ${GIT_LIST} -maxdepth 1 -name "${OS}_helm.inc"
-}
-
-HELM_FILES=$(get_helm_files)
-if [ -z "$(echo -n ${HELM_FILES})" ]; then
-    echo "Could not find ${OS}_helm.inc files" >&2
+# Validate application
+APP_REPO=${MY_REPO}/stx/stx-config/kubernetes/applications/
+if [ ! -d ${APP_REPO} ];then
+    echo "Unable to find the applications directory: ${APP_REPO}" >&2
+    exit 1
+fi
+AVALIABLE_APPS=($(ls ${APP_REPO}))
+if [ ${#AVALIABLE_APPS[@]} -eq 0 ]; then
+    echo "No application found" >&2
+    exit 1
+fi
+if ! is_in ${APP_NAME} ${AVALIABLE_APPS[@]}; then
+    echo "Invalid application: ${APP_NAME}" >&2
     exit 1
 fi
 
+# Cleanup the previous chart build workspace
 BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-helm/stx
 if [ -d ${BUILD_OUTPUT_PATH} ]; then
     # Wipe out the existing dir to ensure there are no stale files
     rm -rf ${BUILD_OUTPUT_PATH}
+    if [ $? -ne 0 ]; then
+        echo "Failed to cleanup the workspace ${BUILD_OUTPUT_PATH}" >&2
+        exit 1
+    fi
 fi
 mkdir -p ${BUILD_OUTPUT_PATH}
+if [ $? -ne 0 ]; then
+    echo "Failed to create the workspace ${BUILD_OUTPUT_PATH}" >&2
+    exit 1
+fi
 cd ${BUILD_OUTPUT_PATH}
 
-IMAGE_FILE_PATH=${BUILD_OUTPUT_PATH}/image_file
-if [ ${#IMAGE_FILES[@]} -ne 0 ]; then
-    mkdir ${IMAGE_FILE_PATH}
+# Create a directory to store image record files
+IMAGE_RECORD_PATH=${BUILD_OUTPUT_PATH}/image_record
+if [ ${#IMAGE_RECORDS[@]} -ne 0 ]; then
+    mkdir -p ${IMAGE_RECORD_PATH}
+    if [ $? -ne 0 ]; then
+        echo "Failed to create the ${IMAGE_RECORD_PATH}" >&2
+        exit 1
+    fi
 fi
 
-# Read the image versions from the passed image
-# files and build them into armada manifest
-function build_image_versions_to_manifest {
-    local manifest_file=$1
-
-    for image_file in ${IMAGE_FILES[@]}; do
-
-        if [[ ${image_file} =~ ^https?://.*(.lst|.txt)$ ]]; then
-            wget --quiet --no-clobber ${image_file} \
-                 --directory-prefix ${IMAGE_FILE_PATH}
-
-            if [ $? -ne 0 ]; then
-                echo "Failed to download image file from ${image_file}" >&2
-                exit 1
-            fi
-        elif [[ -f ${image_file} && ${image_file} =~ .lst|.txt ]]; then
-            cp ${image_file} ${IMAGE_FILE_PATH}
-        else
-            echo "Cannot recognize the provided image file:${image_file}" >&2
-            exit 1
-        fi
-
-        image_file=${IMAGE_FILE_PATH}/${image_file##*/}
-        for image_pattern in $(sed -e 's#/#\\/#g' ${image_file}); do
-
-            # Extract image name from the input image file and
-            image_name=$(echo ${image_pattern} | sed -n 's/.*\/\(.*\):.*$/\1/p')
-
-            # Replace the old image with the new image in manifest file
-            old_image_pattern="\([a-zA-Z0-9.]*\|[0-9.:]*\)\/.*${image_name}:.*"
-            sed -i "s/${old_image_pattern}/${image_pattern}/" ${manifest_file}
-
-            if [ $? -ne 0 ]; then
-                echo "Failed to update manifest file" >&2
-                exit 1
-            fi
-        done
-    done
-}
-
-# Extract the helm charts, order does not matter.
-declare -a FAILED
+# Extract helm charts and app version from the application rpm
+APP_RPM=${APP_NAME}-helm
 RPMS_DIR=${MY_WORKSPACE}/std/rpmbuild/RPMS
 GREP_GLOB="-[^-]*-[^-]*.tis.noarch.rpm"
-for helm_rpm in $(sed -e 's/#.*//' ${HELM_FILES} | sort -u); do
-    case $OS in
-        centos)
-            # Bash globbing does not handle [^-] like regex
-            # so grep needed to be used
-            rpm_file=$(ls ${RPMS_DIR} | grep "^${helm_rpm}${GREP_GLOB}")
-            chartfile=${RPMS_DIR}/${rpm_file}
-            if [ ! -f ${chartfile} ]; then
-                echo "Could not find ${helm_rpm}" >&2
-                FAILED+=($helm_rpm)
-                continue
-            fi
-
-            rpm2cpio ${chartfile} | cpio ${CPIO_FLAGS}
-            if [ ${PIPESTATUS[0]} -ne 0 -o ${PIPESTATUS[1]} -ne 0 ]; then
-                echo "Failed to extract content of ${chartfile}" >&2
-                FAILED+=($helm_rpm)
-            fi
-
-            ;;
-    esac
-done
-
-if [ ${#FAILED[@]} -gt 0 ]; then
-    echo "Failed to find or extract one or more helm packages:" >&2
-    for helm_rpm in ${FAILED[@]}; do
-        echo "${helm_rpm}" >&2
+extract_application_rpm
+# Extract helm charts from the application dependent rpms
+if [ ${#APP_HELM_FILES[@]} -gt 0 ]; then
+    for helm_rpm in ${APP_HELM_FILES[@]}; do
+        extract_chartfile ${helm_rpm}
     done
-    exit 1
 fi
 
 # Create a new tarball containing all the contents we extracted
 # tgz files under helm are relocated to subdir charts.
 # Files under armada are left at the top level
-mkdir staging
+mkdir -p staging
+if [ $? -ne 0 ]; then
+    echo "Failed to create ${BUILD_OUTPUT_PATH}/staging" >&2
+    exit 1
+fi
 
 if [ ! -d "usr/lib/armada" ] || [ ! -d "usr/lib/helm" ]; then
     echo "Failed to create the tarball. Mandatory files are missing." >&2
@@ -215,33 +337,65 @@ fi
 
 # Stage all the charts
 cp -R usr/lib/helm staging/charts
+if [ $? -ne 0 ]; then
+    echo "Failed to copy the charts from ${BUILD_OUTPUT_PATH}/usr/lib/helm to ${BUILD_OUTPUT_PATH}/staging/charts" >&2
+    exit 1
+fi
 
 # Build tarballs for each armada yaml file
 echo "Results:"
 for manifest in usr/lib/armada/*.yaml; do
-    build_image_versions_to_manifest ${manifest}
-    cp ${manifest} staging/.
-    manifest_file=${manifest##*/}
+    manifest_file=$(basename ${manifest})
     manifest_name=${manifest_file%.yaml}
-    # Add an md5
-    cd staging
-    find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5
-    cd ..
+    deprecated_tarball_name="helm-charts-${manifest_name}"
+    build_image_versions_to_manifest ${manifest}
 
-    if [ -n "${LABEL}" ]; then
-        tarball_name="helm-charts-${manifest_name}-${LABEL}.tgz"
-    else
-        tarball_name="helm-charts-${manifest_name}.tgz"
+    cp ${manifest} staging/.
+    if [ $? -ne 0 ]; then
+        echo "Failed to copy the manifests to ${BUILD_OUTPUT_PATH}/staging" >&2
+        exit 1
     fi
 
+    cd staging
+    # Add metadata file
+    touch metadata.yaml
+    if [ -n "${LABEL}" ]; then
+        APP_VERSION=${APP_VERSION}-${LABEL}
+        deprecated_tarball_name=${deprecated_tarball_name}-${LABEL}
+    fi
+    echo "app_name: ${APP_NAME}" >> metadata.yaml
+    echo "app_version: ${APP_VERSION}" >> metadata.yaml
+    if [ -n "${PATCH_DEPENDENCIES}" ]; then
+        echo "patch_dependencies:" >> metadata.yaml
+        for patch in ${PATCH_DEPENDENCIES[@]}; do
+            echo "  - ${patch}" >> metadata.yaml
+        done
+    fi
+
+    # Add an md5
+    find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5
+
+    cd ..
+    tarball_name="${APP_NAME}-${APP_VERSION}.tgz"
     tar ${TAR_FLAGS} ${tarball_name} -C staging/ .
     if [ $? -ne 0 ]; then
         echo "Failed to create the tarball" >&2
         exit 1
     fi
+
     rm staging/${manifest_file}
     rm staging/checksum.md5
     echo "    ${BUILD_OUTPUT_PATH}/${tarball_name}"
+
+    # Create a symbolic link to point to the generated tarball
+    # TODO: Remove the symboblic link once the community has an
+    # opportunity to adapt to the changes in filenames
+    if [ "${APP_NAME}" = "stx-openstack" ]; then
+        ln -s ${BUILD_OUTPUT_PATH}/${tarball_name} ${BUILD_OUTPUT_PATH}/${deprecated_tarball_name}.tgz
+        echo "    ${BUILD_OUTPUT_PATH}/${deprecated_tarball_name}.tgz"
+        echo "Warning: The tarball ${deprecated_tarball_name}.tgz is a symbolic link for ${tarball_name}. It's deprecated and will be removed shortly."
+    fi
+
 done
 
 exit 0