Merge "Fast download of lower layer rpms"

centos-mirror-tools/dl_lower_layer_rpms.sh

@@ -0,0 +1,529 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+#
+# Fast download of StarlingX built rpms using verifytree and repsync
+#
+
+DL_LOWER_LAYER_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
+
+source $DL_LOWER_LAYER_RPMS_DIR/utils.sh
+
+usage() {
+    echo "$0 -l <layer> -b <build_type> -r <url> <rpms_list> <match_level> [ -c <yum.conf> ] [-D <distro> ] [-s|-S|-u|-U] [-x]"
+    echo ""
+    echo "Options:"
+    echo "  -b: <build type>:  e.g. std, rt, installer."
+    echo "  -c: Use an alternate yum.conf rather than the system file"
+    echo "  -l: <layer>: e.g. compiler, distro, flock."
+    echo "  -r: <url>: Url of the root of the repo.  Expect to find a repodata dir there."
+    echo "  -x: Clean log files only, do not run."
+    echo "  rpm_list: a list of RPM files to be downloaded."
+    echo "  match_level: value could be L1, L2 or L3:"
+    echo "    L1: use name, major version and minor version:"
+    echo "        vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm"
+    echo "    L2: use name and major version:"
+    echo "        using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
+    echo "    L3: use name:"
+    echo "        using vim to search vim-7.4.160-2.el7.src.rpm"
+    echo "    K1: Use Koji rather than yum repos as a source."
+    echo "        Koji has a longer retention period than epel mirrors."
+    echo ""
+    echo "  Download Source Options:  Only select one of these."
+    echo "    -s: Download from StarlingX mirror only"
+    echo "    -S: Download from StarlingX mirror, upstream as backup (default)"
+    echo "    -u: Download from original upstream sources only"
+    echo "    -U: Download from original upstream sources, StarlingX mirror as backup"
+    echo ""
+    echo "Returns: 0 = All files downloaded successfully"
+    echo "         1 = Some files could not be downloaded"
+    echo "         2 = Bad arguements or other error"
+    echo ""
+}
+
+
+CLEAN_LOGS_ONLY=0
+dl_rc=0
+
+# Permitted values of dl_source
+dl_from_stx_mirror="stx_mirror"
+dl_from_upstream="upstream"
+dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream"
+dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror"
+
+# Download from what source?
+#   dl_from_stx_mirror = StarlingX mirror only
+#   dl_from_upstream   = Original upstream source only
+#   dl_from_stx_then_upstream = Either source, STX prefered (default)"
+#   dl_from_upstream_then_stx = Either source, UPSTREAM prefered"
+dl_source="$dl_from_stx_then_upstream"
+dl_flag=""
+build_type=""
+lower_layer=""
+url_root=""
+distro="centos"
+
+# Set a default yum.conf which can be overridden by use of '-c' option.
+# I assume we are called from download_mirror.sh and are already in
+# stx-tools/centos-mirror-tools directory.
+YUM_CONF=yum.conf.sample
+
+MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U"
+
+multiple_dl_flag_check () {
+    if [ "$dl_flag" != "" ]; then
+        echo "$MULTIPLE_DL_FLAG_ERROR_MSG"
+        usage
+        exit 1
+    fi
+}
+
+# Parse option flags
+while getopts "b:c:l:D:hr:sSuUx" o; do
+    case "${o}" in
+        b)
+            build_type="$OPTARG"
+            ;;
+        c)
+            # Use an alternate yum.conf
+            YUM_CONF="$OPTARG"
+            ;;
+        D)
+            distro="${OPTARG}"
+            ;;
+        l)
+            lower_layer="$OPTARG"
+            ;;
+        r)
+            # URL
+            url_root="${OPTARG}"
+            ;;
+        s)
+            # Download from StarlingX mirror only. Do not use upstream sources.
+            multiple_dl_flag_check
+            dl_source="$dl_from_stx_mirror"
+            dl_flag="-s"
+            ;;
+        S)
+            # Download from StarlingX mirror first, only use upstream source as a fallback.
+            multiple_dl_flag_check
+            dl_source="$dl_from_stx_then_upstream"
+            dl_flag="-S"
+            ;;
+        u)
+            # Download from upstream only. Do not use StarlingX mirror.
+            multiple_dl_flag_check
+            dl_source="$dl_from_upstream"
+            dl_flag="-u"
+            ;;
+        U)
+            # Download from upstream first, only use StarlingX mirror as a fallback.
+            multiple_dl_flag_check
+            dl_source="$dl_from_upstream_then_stx"
+            dl_flag="-U"
+            ;;
+        x)
+            # Clean only
+            CLEAN_LOGS_ONLY=1
+            ;;
+        h)
+            # Help
+            usage
+            exit 0
+            ;;
+        *)
+            usage
+            exit 2
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "${build_type}" ] || [ -z "${lower_layer}" ] || [ -z "${url_root}" ]; then
+    usage
+    exit 2
+fi
+
+if [ $# -lt 2 ]; then
+    usage
+    exit 2
+fi
+
+if [ "$1" == "" ]; then
+    echo "Need to supply the rpm file list"
+    exit 2;
+else
+    rpms_list=$1
+    echo "using $rpms_list as the download name lists"
+fi
+
+match_level="L1"
+
+if [ ! -z "$2" -a "$2" != " " ];then
+    match_level=$2
+fi
+
+if [ ! -f ${YUM_CONF} ]; then
+    echo "ERROR: failed to find ${YUM_CONF}"
+    usage
+    exit 2
+fi
+
+timestamp=$(date +%F_%H%M)
+echo $timestamp
+
+export DL_MIRROR_LOG_DIR="${DL_MIRROR_LOG_DIR:-./logs}"
+export DL_MIRROR_OUTPUT_DIR="${DL_MIRROR_OUTPUT_DIR:-./output/stx-r1/CentOS/pike}"
+
+dl_dir="$(readlink -f ${DL_MIRROR_OUTPUT_DIR})/layer_repos/${lower_layer}/${build_type}"
+
+LOGSDIR="${DL_MIRROR_LOG_DIR}"
+MISSING_RPMS="$LOGSDIR/${match_level}_rpms_missing_${lower_layer}_${build_type}.log"
+FOUND_RPMS="$LOGSDIR/${match_level}_rpms_found_${lower_layer}_${build_type}.log"
+cat /dev/null > $MISSING_RPMS
+cat /dev/null > $FOUND_RPMS
+
+
+if [ $CLEAN_LOGS_ONLY -eq 1 ];then
+    exit 0
+fi
+
+CREATEREPO=$(which createrepo_c)
+if [ $? -ne 0 ]; then
+    CREATEREPO="createrepo"
+fi
+
+number_of_cpus () {
+    /usr/bin/nproc
+}
+
+get_remote_dir () {
+    local url="${1}"
+    local dest_dir="${2}"
+    mkdir -p "${dest_dir}" || return 1
+    \rm "${dest_dir}/"index.html*
+    wget  -c -N --recursive --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}/"
+}
+
+get_remote_file () {
+    local url="${1}"
+    local dest_dir="${2}"
+    mkdir -p "${dest_dir}" || return 1
+    wget  -c -N --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}"
+}
+
+get_remote_file_overwrite () {
+    local url="${1}"
+    local dest_dir="${2}"
+    local dest_file="${dest_dir}/$(basename ${url})"
+    mkdir -p "${dest_dir}" || return 1
+
+    if [ -f "${dest_file}" ]; then
+        \rm "${dest_file}"
+    fi
+    wget  -c -N --no-parent --no-host-directories --no-directories --directory-prefix="${dest_dir}" "${url}"
+}
+
+clean_repodata () {
+    local repodata="${1}"
+    local f=""
+    local f2=""
+
+    if [ ! -f "${repodata}/repomd.xml" ]; then
+        echo "Error: clean_repodata: file not found: ${repodata}/repomd.xml"
+        return 1
+    fi
+
+    for f in $(find "${repodata}" -name '[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]*'); do
+        f2=$(basename "${f}")
+        if ! grep -q "${f2}" "${repodata}/repomd.xml"; then
+            \rm "${f}"
+        fi
+    done
+}
+
+
+dl_repo () {
+    local base_url="${1}"
+    local dl_dir="${2}"
+    local rpms_list="${3}"
+
+    if [ -z "${base_url}" ] || [ -z "${dl_dir}" ] || [ -z "${rpms_list}" ]; then
+        echo "ERROR: dl_repo: missing arguement"
+        return 1
+    fi
+
+    if [ ! -f "${rpms_list}" ]; then
+        echo "ERROR: dl_repo: no such file '${rpms_list}'"
+        return 1
+    fi
+
+    local REPO_URL="${base_url}"
+    local DOWNLOAD_PATH="${dl_dir}"
+    local DOWNLOAD_PATH_NEW="${DOWNLOAD_PATH}.new"
+    local DOWNLOAD_PATH_OLD="${DOWNLOAD_PATH}.old"
+
+    mkdir -p "${DOWNLOAD_PATH}"
+
+    local YUM_CONF_TMP
+    local TMP
+    local YUM_CONF_DIR_TMP
+    local MUNGED_LIST
+    YUM_CONF_TMP="$(mktemp "/tmp/${YUM_CONF}.XXXXXX")"
+    TMP=$(echo "${YUM_CONF_TMP}" | sed "s#^.*${YUM_CONF}.##")
+    YUM_CONF_DIR=$(dirname "${YUM_CONF_TMP}")
+    YUM_REPOS_DIR_TMP="${YUM_CONF_DIR}/yum.repos.d.${TMP}"
+    MUNGED_LIST="${YUM_CONF_DIR}/yum.lst.${TMP}"
+
+    grep -v '^$' "${rpms_list}" | grep -v '^#' | sed 's#^\(.*\)[.]rpm#\t\1#' | sort --unique > ${MUNGED_LIST}
+    \cp "${YUM_CONF}" "${YUM_CONF_TMP}"
+    sed -i "s#^reposdir=.*#reposdir=${YUM_REPOS_DIR_TMP}#" "${YUM_CONF_TMP}"
+    mkdir -p "${YUM_REPOS_DIR_TMP}"
+
+    REPOID=${lower_layer}_${build_type}
+    REPO_FILE="${YUM_REPOS_DIR_TMP}/${REPOID}.repo"
+    echo "[${REPOID}]" > "${REPO_FILE}"
+    echo "name=${REPOID}" >> "${REPO_FILE}"
+    echo "baseurl=${REPO_URL}" >> "${REPO_FILE}"
+    echo "includepkgs=" >> "${REPO_FILE}"
+    echo "include=file://${MUNGED_LIST}" >> "${REPO_FILE}"
+    echo "enabled=0" >> "${REPO_FILE}"
+
+    # copy repo to a temp location
+    if [ -d "${DOWNLOAD_PATH_NEW}" ]; then
+        \rm -rf "${DOWNLOAD_PATH_NEW}"
+    fi
+
+    if [ -d "${DOWNLOAD_PATH}" ]; then
+        CMD="\cp --archive --link '${DOWNLOAD_PATH}' '${DOWNLOAD_PATH_NEW}'"
+        echo "$CMD"
+        eval $CMD
+        if [ $? -ne 0 ]; then
+            echo "Error: $CMD"
+            return 1
+        fi
+    fi
+
+    #  Download latest repodata
+    get_remote_dir "${REPO_URL}/repodata" "${DOWNLOAD_PATH_NEW}/repodata.upstream"
+    if [ $? -ne 0 ]; then
+        echo "Error: get_remote_dir ${REPO_URL}/repodata ${DOWNLOAD_PATH_NEW}/repodata.upstream"
+        return 1
+    fi
+
+    get_remote_file_overwrite "${REPO_URL}/repodata/repomd.xml" "${DOWNLOAD_PATH_NEW}/repodata.upstream/"
+    if [ $? -ne 0 ]; then
+        echo "Error: get_remote_file_overwrite ${REPO_URL}/repodata/repomd.xml ${DOWNLOAD_PATH_NEW}/repodata.upstream/"
+        return 1
+    fi
+
+    clean_repodata "${DOWNLOAD_PATH_NEW}/repodata.upstream/"
+
+    #  Download latest rpm.lst
+    get_remote_file_overwrite "${REPO_URL}/rpm.lst" "${DOWNLOAD_PATH_NEW}/"
+
+    #
+    # Delete rpms that are no longer valid
+    #
+
+    # Save active repodata as local
+    if [ -d "${DOWNLOAD_PATH_NEW}/repodata" ]; then
+        CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata' '${DOWNLOAD_PATH_NEW}/repodata.local'"
+        echo "$CMD"
+        eval $CMD
+        if [ $? -ne 0 ]; then
+            echo "Error: $CMD"
+            return 1
+        fi
+    fi
+
+    # Make upstream repodata the active copy
+    CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata.upstream' '${DOWNLOAD_PATH_NEW}/repodata'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        return 1
+    fi
+
+    # Do the audit, delete anything broken
+    for f in $(verifytree -a "file://${DOWNLOAD_PATH_NEW}" | \
+                    sed '1,/Checking all packages/d' | \
+                    grep -v ' FAILED$' | \
+                    awk '{ print $2 }' | \
+                    sed 's/^[0-9]*://'); do
+        echo "Already have $f"
+    done
+    for f in $(verifytree -a "file://${DOWNLOAD_PATH_NEW}" | \
+                    sed '1,/Checking all packages/d' | \
+                    grep ' FAILED$' | \
+                    awk '{ print $2 }' | \
+                    sed 's/^[0-9]*://'); do
+        echo "Downloading $f"
+        for f_path in $(find "${DOWNLOAD_PATH_NEW}" -name ${f}.rpm); do
+            CMD="\rm '${f_path}'"
+            echo "$CMD"
+            eval $CMD
+            if [ $? -ne 0 ]; then
+                echo "Error: $CMD"
+                return 1
+            fi
+        done
+    done
+
+    # deactivate and restore upstream repo data
+    CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata' '${DOWNLOAD_PATH_NEW}/repodata.upstream'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        return 1
+    fi
+
+    # Restore our active repodata
+    if [ -d "${DOWNLOAD_PATH_NEW}/repodata.local" ]; then
+        CMD="\mv '${DOWNLOAD_PATH_NEW}/repodata.local' '${DOWNLOAD_PATH_NEW}/repodata'"
+        echo "$CMD"
+        eval $CMD
+        if [ $? -ne 0 ]; then
+            echo "Error: $CMD"
+            return 1
+        fi
+    fi
+
+    # Sync the repo's rpms
+    CMD="reposync --tempcache --norepopath -l --config='${YUM_CONF_TMP}' --repoid=$REPOID --download_path='${DOWNLOAD_PATH_NEW}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        return 1
+    fi
+
+    CMD="pushd '${DOWNLOAD_PATH_NEW}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        return 1
+    fi
+
+    # Update the repodata
+    OPTIONS="--workers $(number_of_cpus)"
+    if [ -f comps.xml ]; then
+        OPTIONS="$OPTIONS -g comps.xml"
+    fi
+    if [ -d repodata ]; then
+        OPTIONS="$OPTIONS --update"
+    fi
+
+    CMD="$CREATEREPO $OPTIONS ."
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        popd
+        return 1
+    fi
+
+    popd
+
+    # Swap out the old copy of our repo
+    if [ -d "${DOWNLOAD_PATH}" ]; then
+        CMD="\mv '${DOWNLOAD_PATH}' '${DOWNLOAD_PATH_OLD}'"
+        echo "$CMD"
+        eval $CMD
+        if [ $? -ne 0 ]; then
+            echo "Error: $CMD"
+            \rm -rf "${DOWNLOAD_PATH_NEW}"
+            return 1
+        fi
+    fi
+
+    # Swap in the updated repo
+    CMD="\mv '${DOWNLOAD_PATH_NEW}' '${DOWNLOAD_PATH}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+        \mv "${DOWNLOAD_PATH_NEW}" "${DOWNLOAD_PATH}"
+        return 1
+    fi
+
+    # Delete the old repo
+    if [ -d "${DOWNLOAD_PATH_OLD}" ]; then
+        CMD="\rm -rf '${DOWNLOAD_PATH_OLD}'"
+        echo "$CMD"
+        eval $CMD
+        if [ $? -ne 0 ]; then
+            echo "Error: $CMD"
+        fi
+    fi
+
+    CMD="\rm '${YUM_CONF_TMP}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+    fi
+
+    CMD="\rm '${MUNGED_LIST}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+    fi
+
+    CMD="\rm -rf '${YUM_REPOS_DIR_TMP}'"
+    echo "$CMD"
+    eval $CMD
+    if [ $? -ne 0 ]; then
+        echo "Error: $CMD"
+    fi
+
+    return 0
+}
+
+#
+# Loop over download sources... typically the STX mirror folowed by upstream
+# ... until we have all the rpms.
+#
+
+RC=1
+for dl_src in $dl_source; do
+    url_root_to_use="${url_root}"
+    case $dl_src in
+        $dl_from_stx_mirror)
+            url_root_to_use="$(url_to_stx_mirror_url "${url_root}" ${distro})"
+            ;;
+        $dl_from_upstream)
+            url_root_to_use="${url_root}"
+            ;;
+        *)
+            echo "Error: Unknown dl_source '$dl_src'"
+            continue
+            ;;
+    esac
+
+    dl_repo "${url_root_to_use}"  "${dl_dir}"  "${rpms_list}"
+    if [ $? -eq 0 ]; then
+        RC=0
+        break
+    fi
+done
+
+for rpm_name in $(grep -v '^$' "${rpms_list}" | grep -v '^#' ); do
+    if [ ! -f "${dl_dir}/${rpm_name}" ]; then
+        echo "${rpm_name}" >> $MISSING_RPMS
+        echo "Failed to download: ${rpm_name}"
+        RC=1
+    fi
+done
+
+
+exit $RC

centos-mirror-tools/dl_rpms.sh

@@ -20,7 +20,7 @@ DL_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
 source $DL_RPMS_DIR/utils.sh
 
 usage() {
-    echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
+    echo "$0 [-n] [-c <yum.conf>] [-s|-S|-u|-U] [-x] <rpms_list> <match_level> "
     echo ""
     echo "Options:"
     echo "  -n: Do not use sudo when performing operations"
@@ -37,6 +37,12 @@ usage() {
     echo "    K1: Use Koji rather than yum repos as a source."
     echo "        Koji has a longer retention period than epel mirrors."
     echo ""
+    echo "  Download Source Options:  Only select one of these."
+    echo "    -s: Download from StarlingX mirror only"
+    echo "    -S: Download from StarlingX mirror, upstream as backup (default)"
+    echo "    -u: Download from original upstream sources only"
+    echo "    -U: Download from original upstream sources, StarlingX mirror as backup"
+    echo ""
     echo "Returns: 0 = All files downloaded successfully"
     echo "         1 = Some files could not be downloaded"
     echo "         2 = Bad arguements or other error"
@@ -100,7 +106,7 @@ while getopts "c:nxD:sSuUh" o; do
             dl_flag="-s"
             ;;
         S)
-            # Download from StarlingX mirror only. Do not use upstream sources.
+            # Download from StarlingX mirror first, only use upstream source as a fallback.
             multiple_dl_flag_check
             dl_source="$dl_from_stx_then_upstream"
             dl_flag="-S"
@@ -112,7 +118,7 @@ while getopts "c:nxD:sSuUh" o; do
             dl_flag="-u"
             ;;
         U)
-            # Download from upstream only. Do not use StarlingX mirror.
+            # Download from upstream first, only use StarlingX mirror as a fallback.
             multiple_dl_flag_check
             dl_source="$dl_from_upstream_then_stx"
             dl_flag="-U"

centos-mirror-tools/download_mirror.sh

@@ -82,6 +82,7 @@ need_dir(){
 
 # Downloader scripts
 rpm_downloader="${DOWNLOAD_MIRROR_DIR}/dl_rpms.sh"
+lower_layer_rpm_downloader="${DOWNLOAD_MIRROR_DIR}/dl_lower_layer_rpms.sh"
 rpm_from_url_downloader="${DOWNLOAD_MIRROR_DIR}/dl_rpms_from_url.sh"
 tarball_downloader="${DOWNLOAD_MIRROR_DIR}/dl_tarball.sh"
 other_downloader="${DOWNLOAD_MIRROR_DIR}/dl_other_from_centos_repo.sh"
@@ -102,6 +103,7 @@ rpms_from_3rd_parties_template="rpms_3rdparties.lst"
 rpms_from_centos_repo_template="rpms_centos.lst"
 rpms_from_centos_3rd_parties_template="rpms_centos3rdparties.lst"
 rpms_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_pkg_lists
+rpms_from_layer_repos_dir=${DL_MIRROR_OUTPUT_DIR}/layer_repos
 image_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_image_inc
 build_info_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_build_info
 tarball_downloads_template="tarball-dl.lst"
@@ -190,7 +192,7 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
             dl_flag="-s"
             ;;
         S)
-            # Download from StarlingX mirror only. Do not use upstream sources.
+            # Download from StarlingX mirror first, only use upstream source as a fallback.
             multiple_dl_flag_check
             dl_source="$dl_from_stx_then_upstream"
             dl_flag="-S"
@@ -202,7 +204,7 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do
             dl_flag="-u"
             ;;
         U)
-            # Download from upstream only. Do not use StarlingX mirror.
+            # Download from upstream first, only use StarlingX mirror as a fallback.
             multiple_dl_flag_check
             dl_source="$dl_from_upstream_then_stx"
             dl_flag="-U"
@@ -324,7 +326,7 @@ for key in "${!layer_image_inc_urls[@]}"; do
     url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' )
     name_from_url=$(url_to_file_name $url)
     dest="${build_info_from_layer_build_dir}/${name_from_url}"
-    curl --silent --fail ${url} > ${dest} ||
+    curl --silent --fail ${url} > ${dest}
     if [ $? -ne 0 ]; then
         echo "ERROR: Failed to download from url: ${url}"
         exit 1
@@ -432,7 +434,12 @@ for key in "${!layer_pkg_urls[@]}"; do
         #download RPMs/SRPMs from CentOS repos by "yumdownloader"
         level=L1
         logfile=$(generate_log_name $list $level)
-        $rpm_downloader ${rpm_downloader_extra_args} $list $level |& tee $logfile
+        llrd_extra_args=""
+        if [ $use_system_yum_conf -eq 0 ]; then
+            llrd_extra_args="-c '${alternate_yum_conf}'"
+        fi
+        echo "$lower_layer_rpm_downloader -l ${lower_layer} -b ${build_type} -r $(dirname $url) ${llrd_extra_args} ${list} ${level}"
+        $lower_layer_rpm_downloader -l ${lower_layer} -b ${build_type} -r $(dirname $url) ${llrd_extra_args} ${list} ${level} |& tee $logfile
         local_retcode=${PIPESTATUS[0]}
     fi
 
@@ -583,7 +590,9 @@ fi
 
 if [ $change_group_ids -eq 1 ]; then
     # change "./output" and sub-folders to 751 (cgcs) group
-    ${SUDO} chown  751:751 -R ./output
+    NEW_UID=$(id -u)
+    NEW_GID=751
+    ${SUDO} chown  ${NEW_UID}:${NEW_GID} -R ./output
 fi
 
 echo "step #4: start downloading other files ..."

centos-mirror-tools/utils.sh

@@ -66,7 +66,7 @@ get_url() {
             _ret="$(url_to_stx_mirror_url $_ret $distro)"
         fi
     else
-        _url_cmd="$(get_yum_command $_name $_level)"
+        _url_cmd="$(get_yum_command $_name $_level) --url"
 
         # When we add --url to the yum download command,
         # --archlist is no longer enforced.  Multiple
@@ -74,7 +74,7 @@ get_url() {
         # filter urls for the desitered arch.
         local arr=( $(split_filename $_name) )
         local arch=${arr[3]}
-        _ret="$($_url_cmd --url | grep "[.]$arch[.]rpm$")"
+        _ret="$($_url_cmd | grep "[.]$arch[.]rpm$" | head -n 1)"
     fi
     echo "$_ret"
 }

toCOPY/generate-cgcs-centos-repo.sh

@@ -142,7 +142,7 @@ missing_rpms_file=missing.txt
 \rm -f ${missing_rpms_file}
 
 # Strip trailing / from mirror_dir if it was specified...
-mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%")
+mirror_dir=$(readlink -f ${mirror_dir} | sed "s%/$%%")
 
 if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then
     echo "The mirror ${mirror_dir} doesn't has the Binary and Source"