diff --git a/centos-mirror-tools/dl_other_from_centos_repo.sh b/centos-mirror-tools/dl_other_from_centos_repo.sh index 1c5a6060..7ec42320 100755 --- a/centos-mirror-tools/dl_other_from_centos_repo.sh +++ b/centos-mirror-tools/dl_other_from_centos_repo.sh @@ -1,8 +1,93 @@ #!/bin/bash -e -# download non-RPM files from http://vault.centos.org/7.4.1708/os/x86_64/ + +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Download non-RPM files from http://vault.centos.org/7.4.1708/os/x86_64/ +# + +DL_OTHER_FROM_CENTOS_REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_OTHER_FROM_CENTOS_REPO_DIR/url_utils.sh + +usage () { + echo "$0 [-D ] [-s|-S|-u|-U] [-h] []" +} + +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "D:hsSuU" o; do + case "${o}" in + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) if [ $# -lt 2 ]; then - echo "$0 []" + usage exit -1 fi @@ -13,12 +98,15 @@ if [ ! -e $download_list ];then fi save_path=$2 -url_prefix="http://vault.centos.org/7.4.1708/os/x86_64/" -echo "NOTE: please assure Internet access to $url_prefix !!" +upstream_url_prefix="http://vault.centos.org/7.4.1708/os/x86_64/" +stx_mirror_url_prefix="$(url_to_stx_mirror_url "$upstream_url_prefix" "$distro")" + +echo "NOTE: please assure Internet access to $upstream_url_prefix !!" force_update=$3 i=0 +error_count=0 all=`cat $download_list` for ff in $all; do ## skip commented_out item which starts with '#' @@ -30,28 +118,68 @@ for ff in $all; do _name=`echo $ff | cut -d":" -f2-2` if [ "$_type" == "folder" ];then mkdir -p $save_path/$_name + if [ $? -ne 0 ]; then + echo "Error: mkdir -p '$save_path/$_name'" + error_count=$((error_count + 1)) + fi else if [ -e "$save_path/$_name" ]; then echo "Already have $save_path/$_name" continue fi - echo "remote path: $url_prefix/$_name" - echo "local path: $save_path/$_name" - if wget $url_prefix/$_name; then - file_name=`basename $_name` - sub_path=`dirname $_name` - if [ -e "./$file_name" ]; then - let i+=1 - echo "$file_name is downloaded successfully" - mv -f ./$file_name $save_path/$_name - ls -l $save_path/$_name + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url_prefix="$stx_mirror_url_prefix" + ;; + $dl_from_upstream) + url_prefix="$upstream_url_prefix" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + echo "remote path: $url_prefix/$_name" + echo "local path: $save_path/$_name" + if wget $url_prefix/$_name; then + file_name=`basename $_name` + sub_path=`dirname $_name` + if [ -e "./$file_name" ]; then + let i+=1 + echo "$file_name is downloaded successfully" + + \mv -f ./$file_name $save_path/$_name + if [ $? -ne 0 ]; then + echo "Error: mv -f './$file_name' '$save_path/$_name'" + error_count=$((error_count + 1)) + fi + + ls -l $save_path/$_name + fi + break + else + echo "Warning: failed to download $url_prefix/$_name" fi - else - echo "ERROR: failed to download $url_prefix/$_name" + done + + if [ ! -e "$save_path/$_name" ]; then + echo "Error: failed to download '$url_prefix/$_name'" + error_count=$((error_count + 1)) + continue fi fi done +echo "" echo "totally $i files are downloaded!" +if [ $error_count -ne 0 ]; then + echo "" + echo "Encountered $error_count errors" + exit 1 +fi + +exit 0 diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 5b290741..582b3e0f 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -15,7 +15,9 @@ SUDOCMD="sudo -E" RELEASEVER="--releasever=7" YUMCONFOPT="" -source utils.sh +DL_RPMS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_RPMS_DIR/utils.sh usage() { echo "$0 [-n] [-c ] " @@ -45,8 +47,34 @@ usage() { CLEAN_LOGS_ONLY=0 dl_rc=0 +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + # Parse option flags -while getopts "c:nxh" o; do +while getopts "c:nxD:sSuUh" o; do case "${o}" in n) # No-sudo @@ -61,6 +89,35 @@ while getopts "c:nxh" o; do YUMCONFOPT="-c $OPTARG" grep -q "releasever=" $OPTARG && RELEASEVER="--$(grep releasever= ${OPTARG})" ;; + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) # Help usage @@ -124,8 +181,8 @@ fi download () { local _file=$1 local _level=$2 - local _list=$(cat $_file) - local _from=$(get_from $_file) + local _list="" + local _from="" local _arch="" @@ -134,23 +191,51 @@ download () { local download_url="" local rpm_name="" local SFILE="" + local lvl + local dl_result + + _list=$(cat $_file) + _from=$(get_from $_file) echo "now the rpm will come from: $_from" for ff in $_list; do _arch=$(get_arch_from_rpm $ff) rpm_name="$(get_rpm_name $ff)" - download_cmd="$(get_download_cmd $ff $_level)" dest_dir="$(get_dest_directory $_arch)" if [ ! -e $dest_dir/$rpm_name ]; then - echo "Looking for $rpm_name" - echo "--> run: $download_cmd" - if $download_cmd ; then - download_url="$(get_url $ff $_level)" - SFILE="$(get_rpm_level_name $rpm_name $_level)" - process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" - else - echo "Warning: $rpm_name not found" + dl_result=1 + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + lvl=$dl_from_stx_mirror + ;; + $dl_from_upstream) + lvl=$_level + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + download_cmd="$(get_download_cmd $ff $lvl)" + + echo "Looking for $rpm_name" + echo "--> run: $download_cmd" + if $download_cmd ; then + download_url="$(get_url $ff $lvl)" + SFILE="$(get_rpm_level_name $rpm_name $lvl)" + process_result "$_arch" "$dest_dir" "$download_url" "$SFILE" + dl_result=0 + break + else + echo "Warning: $rpm_name not found" + fi + done + + if [ $dl_result -eq 1 ]; then + echo "Error: $rpm_name not found" echo "missing_srpm:$rpm_name" >> $LOG echo $rpm_name >> $MISSING_SRPMS rc=1 diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index 0658a99b..0c198802 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# +# SPDX-License-Identifier: Apache-2.0 +# + # The build of StarlingX relies, besides RPM Binaries and Sources, in this # repository which is a collection of packages in the form of Tar Compressed # files and 3 RPMs obtained from a Tar Compressed file. This script and a text @@ -14,6 +18,85 @@ script_path="$(dirname $(readlink -f $0))" tarball_file="$script_path/tarball-dl.lst" mvn_artf_file="$script_path/mvn-artifacts.lst" +DL_TARBALL_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source $DL_TARBALL_DIR/url_utils.sh + +usage () { + echo "$0 [-D ] [-s|-S|-u|-U] [-h] []" +} + +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +distro="centos" + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "D:hsSuU" o; do + case "${o}" in + D) + distro="${OPTARG}" + ;; + + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + + if [ ! -e $tarball_file -o ! -e $mvn_artf_file ];then echo "$download_list does not exist, please have a check!" exit -1 @@ -40,20 +123,55 @@ fi # Download function using wget command download_package() { - wget --spider $1 - if [ $? != 0 ]; then - echo "$1 is broken" - else - wget -t 5 --wait=15 $1 + local upstream_url="$1" + local stx_url="" + local url="" + local rc=1 + + stx_url="$(url_to_stx_mirror_url "$upstream_url" "$distro")" + + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$stx_url" + ;; + $dl_from_upstream) + url="$upstream_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + wget --spider "$url" if [ $? != 0 ]; then - echo "$1" > "$output_log" + echo "Warning: '$url' is broken" + else + wget -t 5 --wait=15 "$url" + if [ $? -eq 0 ]; then + rc=0 + break + else + echo "Warning: failed to download '$url'" + continue + fi fi + done + + if [ $rc != 0 ]; then + echo "Error: failed to download '$upstream_url'" + echo "$upstream_url" > "$output_log" fi + + return $rc } # This script will iterate over the tarball.lst text file and execute specific # tasks based on the name of the package: +error_count=0; + for line in $(cat $tarball_file); do # A line from the text file starting with "#" character is ignored @@ -69,11 +187,31 @@ for line in $(cat $tarball_file); do # denotes special handling is required tarball_name=`echo $line | cut -d"#" -f1-1` # - Column 2, name of the directory path after it is decompressed as it is # referenced in the build system recipe. - # - Column 3, the URL for the package download + # - Column 3, the URL for the file or git to download + # - Column 4, download method, one of + # http - download a simple file + # http_filelist - download multiple files by appending a list of subpaths + # to the base url. Tar up the lot. + # http_script - download a simple file, run script whos output is a tarball + # git - download a git, checkout branch and tar it up + # git_script - download a git, checkout branch, run script whos output is a tarball + # + # - Column 5, utility field + # If method is git or git_script, this is a branch,tag,sha we need to checkout + # If method is http_filelist, this is the path to a file containing subpaths. + # Subpaths are appended to the urls and downloaded. + # Otherwise unused + # - Column 6, Path to script. + # Not yet supported. + # Intent is to run this script to produce the final tarball, replacing + # all the special case code currently embedded in this script. tarball_name=$(echo $line | cut -d"#" -f1-1) directory_name=$(echo $line | cut -d"#" -f2-2) tarball_url=$(echo $line | cut -d"#" -f3-3) + method=$(echo $line | cut -d"#" -f4-4) + util=$(echo $line | cut -d"#" -f5-5) + script=$(echo $line | cut -d"#" -f6-6) # Remove leading '!' if present tarball_name="${tarball_name//!/}" @@ -104,6 +242,12 @@ for line in $(cat $tarball_file); do pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz mv linux-tpmdd-e6aef06/security/integrity/ $directory_name tar czvf $tarball_name $directory_name @@ -111,6 +255,12 @@ for line in $(cat $tarball_file); do rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + mkdir $directory_name tar xf $tarball_name --strip-components 1 -C $directory_name rm $tarball_name @@ -122,6 +272,7 @@ for line in $(cat $tarball_file); do popd tar czvf $tarball_name $directory_name rm -rf $directory_name + popd # pushd $directory_name # The mvn.repo.tgz tarball will be created downloading a serie of # of maven artifacts described in mvn-artifacts file. elif [ "$tarball_name" = "mvn.repo.tgz" ]; then @@ -130,11 +281,49 @@ for line in $(cat $tarball_file); do echo "$mvn_artf_file no found" 1>&2 exit 1 fi + + success_all=0 while read -r artf; do echo "download: $(basename $artf)" - wget "$tarball_url/$artf" -P "$directory_name/$(dirname $artf)" + success=1 + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url/$artf" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url/$artf" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + wget "$url" -P "$directory_name/$(dirname $artf)" + if [ $? -eq 0 ]; then + success=0 + break + else + echo "Warning: Failed to download from $url" + continue; + fi + done + + if [ $success -ne 0 ]; then + success_all=1 + echo "Error: Failed to download from '$tarball_url/$artf'" + fi done < "$mvn_artf_file" + if [ $success_all -ne 0 ]; then + echo "Error: Failed to download one or more components from '$tarball_url'" + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + # Create tarball tar -zcvf "$tarball_name" -C "$directory_name"/ . rm -rf "$directory_name" @@ -142,7 +331,14 @@ for line in $(cat $tarball_file); do elif [[ "$tarball_name" =~ ^'MLNX_OFED_LINUX' ]]; then pkg_version=$(echo "$tarball_name" | cut -d "-" -f2-3) srpm_path="MLNX_OFED_SRC-${pkg_version}/SRPMS/" + download_package "$tarball_url" + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar -xf "$tarball_name" tar -xf "$directory_name/src/MLNX_OFED_SRC-${pkg_version}.tgz" # This section of code gets specific SRPMs versions according @@ -168,8 +364,20 @@ for line in $(cat $tarball_file); do rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then download_package $tarball_url + if [ $? -ne 0 ]; then + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name tar czvf $tarball_name $directory_name @@ -177,22 +385,74 @@ for line in $(cat $tarball_file); do rm -rf $directory_name rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then - git clone https://git.code.sf.net/p/ibmtpm20tss/tss ibmtpm20tss-tss - pushd ibmtpm20tss-tss - git checkout v930 + dest_dir=ibmtpm20tss-tss + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + git clone $url $dest_dir + if [ $? -eq 0 ]; then + # Success + break + else + echo "Warning: Failed to git clone from '$url'" + continue + fi + done + + if [ ! -d $dest_dir ]; then + echo "Error: Failed to git clone from '$tarball_url'" + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) + popd # pushd $output_tarball + continue + fi + + pushd $dest_dir + branch=$util + git checkout $branch rm -rf .git popd mv ibmtpm20tss-tss $directory_name tar czvf $tarball_name $directory_name rm -rf $directory_name + popd # pushd $dest_dir fi - popd + popd # pushd $output_tarball continue fi - download_cmd="wget -t 5 --wait=15 $tarball_url -O $download_path" + if [ -e $download_path ]; then + echo "Already have $download_path" + continue + fi + + for dl_src in $dl_source; do + case $dl_src in + $dl_from_stx_mirror) + url="$(url_to_stx_mirror_url "$tarball_url" "$distro")" + ;; + $dl_from_upstream) + url="$tarball_url" + ;; + *) + echo "Error: Unknown dl_source '$dl_src'" + continue + ;; + esac + + download_cmd="wget -t 5 --wait=15 $url -O $download_path" - if [ ! -e $download_path ]; then if $download_cmd ; then echo "Ok: $download_path" pushd $download_directory @@ -204,16 +464,26 @@ for line in $(cat $tarball_file); do rm -r $directory_name fi popd + break else - echo "Error: Failed to download $tarball_url" 1>&2 - echo "$tarball_url" > "$output_log" + echo "Warning: Failed to download $url" 1>&2 + continue fi + done - else - echo "Already have $download_path" + if [ ! -e $download_path ]; then + echo "Error: Failed to download $tarball_url" 1>&2 + echo "$tarball_url" > "$output_log" + error_count=$((error_count + 1)) fi - done # End of file +if [ $error_count -ne 0 ]; then + echo "" + echo "Encountered $error_count errors" + exit 1 +fi + +exit 0 diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 6b6731e1..e55e3d8b 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -4,7 +4,7 @@ # usage() { - echo "$0 [-n] [-c ] [-g]" + echo "$0 [-n] [-c ] [-g] [-s|-S|-u|-U]" echo "" echo "Options:" echo " -n: Do not use sudo when performing operations (option passed on to" @@ -12,6 +12,10 @@ usage() { echo " -c: Use an alternate yum.conf rather than the system file (option passed" echo " on to subscripts when appropriate)" echo " -g: do not change group IDs of downloaded artifacts" + echo " -s: Download from StarlingX mirror only" + echo " -S: Download from StarlingX mirror, upstream as backup (default)" + echo " -u: Download from original upstream sources only" + echo " -U: Download from original upstream sources, StarlingX mirror as backup" echo "" } @@ -24,8 +28,17 @@ generate_log_name() { need_file(){ for f in $*; do - if [ ! -e $f ]; then - echo "ERROR: $f does not exist." + if [ ! -f $f ]; then + echo "ERROR: File $f does not exist." + exit 1 + fi + done +} + +need_dir(){ + for d in $*; do + if [ ! -d $d ]; then + echo "ERROR: Directory $d does not exist." exit 1 fi done @@ -35,12 +48,16 @@ need_file(){ rpm_downloader="./dl_rpms.sh" tarball_downloader="./dl_tarball.sh" other_downloader="./dl_other_from_centos_repo.sh" +make_stx_mirror_yum_conf="./make_stx_mirror_yum_conf.sh" # track optional arguments change_group_ids=1 use_system_yum_conf=1 +alternate_yum_conf="" +alternate_repo_dir="" rpm_downloader_extra_args="" tarball_downloader_extra_args="" +distro="centos" # lst files to use as input rpms_from_3rd_parties="./rpms_3rdparties.lst" @@ -51,8 +68,43 @@ other_downloads="./other_downloads.lst" # Overall success success=1 -# Parse out optional -c or -n arguments -while getopts "c:ngh" o; do +# Permitted values of dl_source +dl_from_stx_mirror="stx_mirror" +dl_from_upstream="upstream" +dl_from_stx_then_upstream="$dl_from_stx_mirror $dl_from_upstream" +dl_from_upstream_then_stx="$dl_from_upstream $dl_from_stx_mirror" + +# Download from what source? +# dl_from_stx_mirror = StarlingX mirror only +# dl_from_upstream = Original upstream source only +# dl_from_stx_then_upstream = Either source, STX prefered (default)" +# dl_from_upstream_then_stx = Either source, UPSTREAM prefered" +dl_source="$dl_from_stx_then_upstream" +dl_flag="" + +dl_from_stx () { + local re="\\b$dl_from_stx_mirror\\b" + [[ "$dl_source" =~ $re ]] +} + +dl_from_upstream () { + local re="\\b$dl_from_upstream\\b" + [[ "$dl_source" =~ $re ]] +} + + +MULTIPLE_DL_FLAG_ERROR_MSG="Error: Please use only one of: -s,-S,-u,-U" + +multiple_dl_flag_check () { + if [ "$dl_flag" != "" ]; then + echo "$MULTIPLE_DL_FLAG_ERROR_MSG" + usage + exit 1 + fi +} + +# Parse out optional arguments +while getopts "c:nghsSuU" o; do case "${o}" in n) # Pass -n ("no-sudo") to rpm downloader @@ -61,12 +113,36 @@ while getopts "c:ngh" o; do c) # Pass -c ("use alternate yum.conf") to rpm downloader use_system_yum_conf=0 - rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}" + alternate_yum_conf="${OPTARG}" ;; g) # Do not attempt to change group IDs on downloaded packages change_group_ids=0 ;; + s) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_mirror" + dl_flag="-s" + ;; + S) + # Download from StarlingX mirror only. Do not use upstream sources. + multiple_dl_flag_check + dl_source="$dl_from_stx_then_upstream" + dl_flag="-S" + ;; + u) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream" + dl_flag="-u" + ;; + U) + # Download from upstream only. Do not use StarlingX mirror. + multiple_dl_flag_check + dl_source="$dl_from_upstream_then_stx" + dl_flag="-U" + ;; h) # Help usage @@ -98,7 +174,6 @@ need_file ${rpms_from_centos_repo} need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst - #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." @@ -111,6 +186,51 @@ if [ ${use_system_yum_conf} -ne 0 ]; then fi fi +if [ $use_system_yum_conf -eq 0 ]; then + need_file "${alternate_yum_conf}" + if [ "$alternate_repo_dir" == "" ]; then + alternate_repo_dir=$(grep '^repodir=' "${alternate_yum_conf}" | cut -d '=' -f 2) + need_dir "${alternate_repo_dir}" + fi +fi + +TEMP_DIR="" +rpm_downloader_extra_args="${rpm_downloader_extra_args} -D $distro" + +if [ "$dl_flag" == "" ]; then + if [ $use_system_yum_conf -eq 0 ]; then + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${alternate_yum_conf}" + fi +else + rpm_downloader_extra_args="${rpm_downloader_extra_args} $dl_flag" + + if ! dl_from_stx; then + if [ $use_system_yum_conf -eq 0 ]; then + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${alternate_yum_conf}" + fi + else + TEMP_DIR=$(mktemp -d /tmp/stx_mirror_XXXXXX) + TEMP_CONF="$TEMP_DIR/yum.conf" + need_file ${make_stx_mirror_yum_conf} + need_dir ${TEMP_DIR} + if [ $use_system_yum_conf -eq 0 ]; then + if dl_from_upstream; then + ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro + else + ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y $alternate_yum_conf -r $alternate_repo_dir -D $distro + fi + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c $TEMP_CONF" + else + if dl_from_upstream; then + ${make_stx_mirror_yum_conf} -R -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro + else + ${make_stx_mirror_yum_conf} -d $TEMP_DIR -y /etc/yum.conf -r /etc/yum.repos.d -D $distro + fi + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c $TEMP_CONF" + fi + fi +fi + list=${rpms_from_3rd_parties} level=L1 logfile=$(generate_log_name $list $level) @@ -139,6 +259,9 @@ fi if [ ${use_system_yum_conf} -eq 1 ]; then # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. \rm -f $REPO_DIR/StarlingX_3rd*.repo + if [ "$TEMP_DIR" != "" ]; then + \rm -f $TEMP_DIR/yum.repos.d/StarlingX_3rd*.repo + fi fi @@ -232,7 +355,7 @@ fi echo "step #3: start downloading other files ..." logfile=$LOGSDIR"/otherfiles_centos_download.log" -${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile +${other_downloader} ${dl_flag} -D "$distro" ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" @@ -249,7 +372,7 @@ fi # they will be downloaded. echo "step #4: start downloading tarball compressed files" logfile=$LOGSDIR"/tarballs_download.log" -${tarball_downloader} ${tarball_downloader_extra_args} |& tee $logfile +${tarball_downloader} ${dl_flag} -D "$distro" ${tarball_downloader_extra_args} |& tee $logfile retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #4: done successfully" @@ -261,6 +384,13 @@ else success=0 fi +# +# Clean up the mktemp directory, if required. +# +if [ "$TEMP_DIR" != "" ]; then + \rm -rf "$TEMP_DIR" +fi + echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" echo "running \"update-pxe-network-installer\" command after \"build-iso\"" diff --git a/centos-mirror-tools/make_stx_mirror_yum_conf.sh b/centos-mirror-tools/make_stx_mirror_yum_conf.sh new file mode 100755 index 00000000..90f5d387 --- /dev/null +++ b/centos-mirror-tools/make_stx_mirror_yum_conf.sh @@ -0,0 +1,226 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# Replicate a yum.conf and yum.repo.d under a temporary directory and +# then modify the files to point to equivalent repos in the StarlingX mirror. +# This script was originated by Scott Little +# + +MAKE_STX_MIRROR_YUM_CONF_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source "$MAKE_STX_MIRROR_YUM_CONF_DIR/url_utils.sh" + +DISTRO="centos" + +TEMP_DIR="" +SRC_REPO_DIR="$MAKE_STX_MIRROR_YUM_CONF_DIR/yum.repos.d" +SRC_YUM_CONF="$MAKE_STX_MIRROR_YUM_CONF_DIR/yum.conf.sample" + +RETAIN_REPODIR=0 + +usage () { + echo "" + echo "$0 -d [-D ] [-y ] [-r ] [-R]" + echo "" + echo "Replicate a yum.conf and yum.repo.d under a new directory and" + echo "then modify the files to point to equivalent repos in the StarlingX" + echo "mirror." + echo "" + echo "-d = Place modified yum.conf and yum.repo.d into this directory" + echo "-D = Target distro on StarlingX mirror. Default is 'centos'" + echo "-y = Path to yum.conf file that we will modify. Default is" + echo " 'yum.conf.sample' in same directory as this script" + echo "-r = Path to yum.repos.d that we will modify. Default is" + echo " 'yum.repos.d' in same directory as this script" +} + + +# +# option processing +# +while getopts "D:d:Rr:y:" o; do + case "${o}" in + D) + DISTRO="${OPTARG}" + ;; + d) + TEMP_DIR="${OPTARG}" + ;; + r) + SRC_REPO_DIR="${OPTARG}" + ;; + R) + RETAIN_REPODIR=1 + ;; + y) + SRC_YUM_CONF="${OPTARG}" + ;; + *) + usage + exit 1 + ;; + esac +done + +# +# option validation +# +if [ ! -f $SRC_YUM_CONF ]; then + echo "Error: yum.conf not found at '$SRC_YUM_CONF'" + exit 1 +fi + +if [ ! -d $SRC_REPO_DIR ]; then + echo "Error: repo dir not found at '$SRC_REPO_DIR'" + exit 1 +fi + +if [ "$TEMP_DIR" == "" ]; then + echo "Error: working dir not provided" + usage + exit 1 +fi + +if [ ! -d $TEMP_DIR ]; then + echo "Error: working dir not found at '$TEMP_DIR'" + exit 1 +fi + +# +# Get the value of the $releasever variable. +# +# If the source yum.conf has a releasever= setting, we will honor +# that, even though yum will not. +# +# Otherwise use yum to query the host environment (Docker). +# This assumes the host environmnet has the same releasever +# as that which will be used inside StarlingX. +# +# NOTE: In other scripts we will read releasever= out of yum.conf +# and push it back into yum via --releasever=<#>. +# +get_releasever () { + if [ -f $SRC_YUM_CONF ] && grep -q '^releasever=' $SRC_YUM_CONF; then + grep '^releasever=' $SRC_YUM_CONF | cut -d '=' -f 2 + else + yum version nogroups | grep Installed | cut -d ' ' -f 2 | cut -d '/' -f 1 + fi +} + +# +# Get the value of the $basearch variable. +# +# Just use yum to query the host environment (Docker) as we don't support +# cross compiling. +# +get_arch () { + yum version nogroups | grep Installed | cut -d ' ' -f 2 | cut -d '/' -f 2 +} + + +# +# Global variables we will use later. +# +CENGN_REPOS_DIR="$TEMP_DIR/yum.repos.d" +CENGN_YUM_CONF="$TEMP_DIR/yum.conf" +CENGN_YUM_LOG="$TEMP_DIR/yum.log" +CENGN_YUM_CACHDIR="$TEMP_DIR/cache/yum/\$basearch/\$releasever" + +RELEASEVER=$(get_releasever) +ARCH=$(get_arch) + +# +# Copy as yet unmodified yum.conf and yum.repos.d from source to dest. +# +echo "\cp -r '$SRC_REPO_DIR' '$CENGN_REPOS_DIR'" +\cp -r "$SRC_REPO_DIR" "$CENGN_REPOS_DIR" +echo "\cp '$SRC_YUM_CONF' '$CENGN_YUM_CONF'" +\cp "$SRC_YUM_CONF" "$CENGN_YUM_CONF" + +# +# Add or modify reposdir= value in our new yum.conf +# +if grep -q '^reposdir=' $CENGN_YUM_CONF; then + # reposdir= already exists, modify it + if [ $RETAIN_REPODIR -eq 1 ]; then + # Append CENGN_REPOS_DIR + sed "s#^reposdir=.*\$#reposdir=\1 $CENGN_REPOS_DIR#" -i $CENGN_YUM_CONF + else + # replace with CENGN_REPOS_DIR + sed "s#^reposdir=.*\$#reposdir=$CENGN_REPOS_DIR#" -i $CENGN_YUM_CONF + fi +else + # reposdir= doeas not yet exist, add it + if [ $RETAIN_REPODIR -eq 1 ]; then + # Add both SRC_REPO_DIR and CENGN_REPOS_DIR + echo "reposdir=$SRC_REPO_DIR $CENGN_REPOS_DIR" >> $CENGN_YUM_CONF + else + # Add CENGN_REPOS_DIR only + echo "reposdir=$CENGN_REPOS_DIR" >> $CENGN_YUM_CONF + fi +fi + +# +# modify or add logfile= value in our new yum.conf +# +if grep -q '^logfile=' $CENGN_YUM_CONF; then + sed "s#^logfile=.*\$#logfile=$CENGN_YUM_LOG#" -i $CENGN_YUM_CONF +else + echo "logfile=$CENGN_YUM_LOG" >> $CENGN_YUM_CONF +fi + +# +# modify or add cachedir= value in our new yum.conf +# +if grep -q '^cachedir=' $CENGN_YUM_CONF; then + sed "s#^cachedir=.*\$#cachedir=$CENGN_YUM_CACHDIR#" -i $CENGN_YUM_CONF +else + echo "cachedir=$CENGN_YUM_CACHDIR" >> $CENGN_YUM_CONF +fi + + +# +# Modify all the repo files in our new yum.repos.d +# +for REPO in $(find "$CENGN_REPOS_DIR" -type f -name '*repo'); do + # + # Replace mirrorlist with baseurl if required + # + if grep -q '^mirrorlist=' "$REPO" ; then + sed '/^mirrorlist=/d' -i "$REPO" + sed 's%^#baseurl%baseurl%' -i "$REPO" + fi + + # + # Substitute any $releasever or $basearch variables + # + sed "s#/[$]releasever/#/$RELEASEVER/#g" -i "$REPO" + sed "s#/[$]basearch/#/$ARCH/#g" -i "$REPO" + + # + # Turn off gpgcheck for now. + # Must revisit this at a later date! + # + sed 's#^gpgcheck=1#gpgcheck=0#' -i "$REPO" + sed '/^gpgkey=/d' -i "$REPO" + + # + # Convert baseurl(s) to cengn equivalent + # + for URL in $(grep '^baseurl=' "$REPO" | sed 's#^baseurl=##'); do + CENGN_URL="$(url_to_stx_mirror_url "$URL" "$DISTRO")" + sed "s#^baseurl=$URL\$#baseurl=$CENGN_URL#" -i "$REPO" + done + + # + # Prefix repoid and name with CENGN + # + sed "s#^name=\(.*\)#name=CENGN_\1#" -i "$REPO" + sed "s#^\[\([^]]*\)\]#[CENGN_\1]#" -i "$REPO" +done + +echo $TEMP_DIR diff --git a/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh b/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh new file mode 100755 index 00000000..cfae2577 --- /dev/null +++ b/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh @@ -0,0 +1,383 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# +# Daily update script for mirror.starlingx.cengn.ca covering +# tarballs and other files not downloaded from a yum repository. +# This script was originated by Scott Little. +# +# IMPORTANT: This script is only to be run on the StarlingX mirror. +# It is not for use by the general StarlinX developer. +# +# This script was originated by Scott Little. +# + +DAILY_DL_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +if [ -f "$DAILY_DL_SYNC_DIR/url_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/url_utils.sh" +elif [ -f "$DAILY_DL_SYNC_DIR/../url_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/../url_utils.sh" +else + echo "Error: Can't find 'url_utils.sh'" + exit 1 +fi + + +LOGFILE=/export/log/daily_dl_sync.log +DOWNLOAD_PATH_ROOT=/export/mirror/centos + +STX_TOOLS_BRANCH="master" +STX_TOOLS_BRANCH_ROOT_DIR="$HOME" +STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git" +STX_TOOLS_OS_SUBDIR="centos-mirror-tools" + +usage () { + echo "$0 [-b ] [-d ]" + echo "" + echo "Options:" + echo " -b: Use an alternate branch of stx-tools. Default is 'master'." + echo " -d: Directory where we will clone stx-tools. Default is \$HOME." + echo "" +} + +while getopts "b:d:h" opt; do + case "${opt}" in + b) + # branch + STX_TOOLS_BRANCH="${OPTARG}" + if [ $"STX_TOOLS_BRANCH" == "" ]; then + usage + exit 1 + fi + ;; + d) + # download directory for stx-tools + STX_TOOLS_BRANCH_ROOT_DIR="${OPTARG}" + if [ "$STX_TOOLS_BRANCH_ROOT_DIR" == "" ]; then + usage + exit 1 + fi + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH" +STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools" +LST_FILE_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR" + +dl_git_from_url () { + local GIT_URL="$1" + local BRANCH="$2" + local DL_DIR="$3" + local DL_ROOT_DIR="" + local SAVE_DIR + local CMD="" + + SAVE_DIR="$(pwd)" + + if [ "$DL_DIR" == "" ]; then + DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')" + fi + + echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'" + DL_ROOT_DIR=$(dirname "$DL_DIR") + + if [ ! -d "$DL_DIR" ]; then + if [ ! -d "$DL_ROOT_DIR" ]; then + CMD="mkdir -p '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + + CMD="cd '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git clone --bare '$GIT_URL' '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git --bare update-server-info" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + if [ -f hooks/post-update.sample ]; then + CMD="mv -f hooks/post-update.sample hooks/post-update" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git fetch" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git checkout '$BRANCH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + cd "$SAVE_DIR" + return 0 +} + + +dl_file_from_url () { + local URL="$1" + local DOWNLOAD_PATH="" + local DOWNLOAD_DIR="" + local PROTOCOL="" + local CMD="" + + DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")" + DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")" + PROTOCOL=$(url_protocol $URL) + echo "$PROTOCOL $URL $DOWNLOAD_PATH" + + if [ -f "$DOWNLOAD_PATH" ]; then + echo "Already have '$DOWNLOAD_PATH'" + return 0 + fi + + case "$PROTOCOL" in + https|http) + if [ ! -d "$DOWNLOAD_DIR" ]; then + CMD="mkdir -p '$DOWNLOAD_DIR'" + echo "$CMD" + eval "$CMD" + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + fi + + CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + ;; + *) + echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'" + ;; + esac + + return 0 +} + + +raw_dl_from_rpm_lst () { + local FILE="$1" + local RPM="" + local URL="" + local ERROR_COUNT=0 + + # Expected format # + grep -v '^#' $FILE | while IFS='#' read -r RPM URL; do + echo "Processing: RPM=$RPM URL=$URL" + dl_file_from_url "$URL" + ERR_COUNT=$((ERR_COUNT+$?)) + done + + return $ERR_COUNT +} + +raw_dl_from_non_rpm_lst () { + local FILE="$1" + local TAR="" + local URL="" + local METHOD="" + local UTIL="" + local SCRIPT="" + local BRANCH="" + local SUBDIRS_FILE="" + local TARBALL_NAME="" + local ERROR_COUNT=0 + + # Expected format ## + # or !####[]#[