starlingx/tools
Code Issues Proposed changes

Use kojipkgs.fedoraproject.org as a backup rpm source.

Browse Source 
EPEL7 rpms builds are visible at kojipkgs.fedoraproject.org
and often remain available there long after they age out
of the official repo.

Use kojipkgs.fedoraproject.org as a backup source for
packages when not found in the repo.  Log seperately
so that these dropouts can be identified and corrected
in the lst files.

Also improved the tests for already downloaded packages
in several places to accelerate the test cycle.

Change-Id: Iaaa9fe05ac605a3604acf0048c571c6e88303692
Story: 2003157
Task: 23292
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2018-08-09 14:22:40 -04:00
parent 6b893381bf
commit fa995d6981
4 changed files with 229 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
centos-mirror-tools/dl_other_from_centos_repo.sh
View File

@ -31,6 +31,11 @@ for ff in $all; do
if [ "$_type" == "folder" ];then
mkdir -p $save_path/$_name
else
if [ -e "$save_path/$_name" ]; then
echo "Already have $save_path/$_name"
continue
fi
echo "remote path: $url_prefix/$_name"
echo "local path: $save_path/$_name"
if wget $url_prefix/$_name; then

145
centos-mirror-tools/dl_rpms.sh
View File

@ -11,6 +11,7 @@ usage() {
echo "Options:"
echo " -n: Do not use sudo when performing operations"
echo " -c: Use an alternate yum.conf rather than the system file"
echo " -x: Clean log files only, do not run."
echo " rpm_list: a list of RPM files to be downloaded."
echo " match_level: value could be L1, L2 or L3:"
echo " L1: use name, major version and minor version:"
@ -19,6 +20,12 @@ usage() {
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
echo " L3: use name:"
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
echo " K1: Use Koji rather than yum repos as a source."
echo " Koji has a longer retention period than epel mirrors."
echo ""
echo "Returns: 0 = All files downloaded successfully"
echo " 1 = Some files could not be downloaded"
echo " 2 = Bad arguements or other error"
echo ""
}
@ -34,13 +41,20 @@ get_from() {
SUDOCMD="sudo -E"
YUMCONFOPT=""
CLEAN_LOGS_ONLY=0
dl_rc=0
# Parse option flags
while getopts "c:nh" o; do
while getopts "c:nxh" o; do
case "${o}" in
n)
# No-sudo
SUDOCMD=""
;;
x)
# Clean only
CLEAN_LOGS_ONLY=1
;;
c)
# Use an alternate yum.conf
YUMCONFOPT="-c $OPTARG"
@ -52,7 +66,7 @@ while getopts "c:nh" o; do
;;
*)
usage
exit 1
exit 2
;;
esac
done
@ -60,12 +74,12 @@ shift $((OPTIND-1))
if [ $# -lt 2 ]; then
usage
exit -1
exit 2
fi
if [ "$1" == "" ]; then
echo "Need to supply the rpm file list"
exit -1;
exit 2;
else
rpms_list=$1
echo "using $rpms_list as the download name lists"
@ -109,50 +123,121 @@ cat /dev/null > $FOUND_RPMS
cat /dev/null > $MISSING_RPMS
cat /dev/null > $URL_RPMS
if [ $CLEAN_LOGS_ONLY -eq 1 ];then
exit 0
fi
# Function to split an rpm filename into parts.
#
# Returns a space seperated list containing:
# <NAME> <VERSION> <RELEASE> <ARCH> <EPOCH>
#
split_filename () {
local rpm_filename=$1
local RPM=""
local SFILE=""
local ARCH=""
local RELEASE=""
local VERSION=""
local NAME=""
RPM=$(echo $rpm_filename | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $rpm_filename | rev | cut -d'.' -f2- | rev)
ARCH=$(echo $SFILE | rev | cut -d'.' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'.' -f2- | rev)
RELEASE=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
SFILE=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
VERSION=$(echo $SFILE | rev | cut -d'-' -f-1 | rev)
NAME=$(echo $SFILE | rev | cut -d'-' -f2- | rev)
if [[ $NAME = *":"* ]]; then
EPOCH=$(echo $NAME | cut -d':' -f-1)
NAME=$(echo $NAME | cut -d':' -f2-)
fi
echo "$NAME" "$VERSION" "$RELEASE" "$ARCH" "$EPOCH"
}
# Function to predict the URL where a rpm might be found.
# Assumes the rpm was compile for EPEL by fedora's koji.
koji_url () {
local rpm_filename=$1
local arr=( $(split_filename $rpm_filename) )
local n=${arr[0]}
local v=${arr[1]}
local r=${arr[2]}
local a=${arr[3]}
local e=${arr[4]}
echo "https://kojipkgs.fedoraproject.org/packages/$n/$v/$r/$a/$n-$v-$r.$a.rpm"
}
# Function to download different types of RPMs in different ways
download () {
_file=$1
_level=$2
_list=$(cat $_file)
_from=$(get_from $_file)
local _file=$1
local _level=$2
local _list=$(cat $_file)
local _from=$(get_from $_file)
local _type=""
local rc=0
local download_cmd=""
local download_url_cmd=""
local rpm_name=""
local rpm_url=""
local SFILE=""
echo "now the rpm will come from: $_from"
for ff in $_list; do
download_cmd=""
download_url_cmd=""
_type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
# Decide if the list will be downloaded using yumdownloader or wget
if [[ $ff != *"#"* ]]; then
rpm_name=$ff
# Cut the rpm name for the specified level (L1, L2 or L3)
if [ $_level == "L1" ]; then
if [ $_level == "K1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $match_level == "L2" ];then
SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
rpm_url=$(koji_url $rpm_name)
download_cmd="wget $rpm_url)"
download_url_cmd="echo $rpm_url)"
else
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo " ------ using $SFILE to search $rpm_name ------"
# Yumdownloader with the appropriate flag for src, noarch or x86_64
if [ "$_type" == "src" ];then
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
else
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
if [ $_level == "L1" ]; then
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
elif [ $match_level == "L2" ];then
SFILE=`echo $rpm_name | rev | cut -d'-' -f2- | rev`
else
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
fi
echo " ------ using $SFILE to search $rpm_name ------"
# Yumdownloader with the appropriate flag for src, noarch or x86_64
if [ "$_type" == "src" ];then
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
else
download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
fi
fi
else
# Buid wget command
rpm_name=`echo $ff | cut -d"#" -f1-1`
rpm_url=`echo $ff | cut -d"#" -f2-2`
download_cmd="wget $rpm_url"
download_url_cmd="echo $rpm_url"
SFILE=$rpm_name
fi
echo "--> run: $download_cmd"
# Put the RPM in the Binary or Source directory
if [ "$_type" == "src" ]; then
if [ ! -e $MDIR_SRC/$rpm_name ]; then
echo "Looking for $rpm_name"
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
@ -165,15 +250,18 @@ download () {
fi
echo $rpm_name >> $FOUND_SRPMS
else
echo "Warning: $rpm_name not found"
echo $rpm_name >> $MISSING_SRPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
echo "Already have ${MDIR_SRC}/${_type}/$rpm_name"
echo $rpm_name >> $FOUND_SRPMS
fi
else ## noarch or x86_64
if [ ! -e ${MDIR_BIN}/${_type}/$rpm_name ]; then
echo "Looking for $rpm_name..."
echo "--> run: $download_cmd"
if $download_cmd ; then
# Success! Record download URL.
# Use 'sort --unique' because sometimes
@ -187,7 +275,9 @@ download () {
fi
echo $rpm_name >> $FOUND_RPMS
else
echo "Warning: $rpm_name not found"
echo $rpm_name >> $MISSING_RPMS
rc=1
fi
else
echo "Already have ${MDIR_BIN}/${_type}/$rpm_name"
@ -195,6 +285,8 @@ download () {
fi
fi
done
return $rc
}
# Prime the cache
@ -204,8 +296,11 @@ ${SUDOCMD} yum ${YUMCONFOPT} makecache
if [ -s "$rpms_list" ];then
echo "--> start searching "$rpms_list
download $rpms_list $match_level
if [ $? -ne 0 ]; then
dl_rc=1
fi
fi
echo "done!!"
exit 0
exit $dl_rc

18
centos-mirror-tools/dl_tarball.sh
View File

@ -75,6 +75,9 @@ for line in $(cat $tarball_file); do
directory_name=$(echo $line | cut -d"#" -f2-2)
tarball_url=$(echo $line | cut -d"#" -f3-3)
# Remove leading '!' if present
tarball_name="${tarball_name//!/}"
# - For the General category and the Puppet category:
# - Packages have a common process: download, decompressed,
# change the directory path and compressed.
@ -87,16 +90,16 @@ for line in $(cat $tarball_file); do
download_directory=$output_tarball
fi
if [ -e $download_path ]; then
echo "Already have $download_path"
continue
fi
# We have 6 packages from the text file starting with the character "!":
# they require special handling besides the common process: remove directory,
# remove text from some files, clone a git repository, etc.
if [[ "$line" =~ ^'!' ]]; then
tarball_name="${tarball_name//!/}"
if [ -e "$output_tarball/$tarball_name" ]; then
echo "Already have $tarball_name"
continue
fi
echo $tarball_name
pushd $output_tarball
if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
@ -157,7 +160,10 @@ for line in $(cat $tarball_file); do
else
echo "$pkg_version : unknown version"
fi
rm -f "$tarball_name"
# Don't delete the original MLNX_OFED_LINUX tarball.
# We don't use it, but it will prevent re-downloading this file.
# rm -f "$tarball_name"
rm -rf "MLNX_OFED_SRC-${pkg_version}"
rm -rf "$directory_name"
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then

107
centos-mirror-tools/download_mirror.sh
View File

@ -31,6 +31,9 @@ rpms_from_centos_repo="./rpms_centos.lst"
rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
other_downloads="./other_downloads.lst"
# Overall success
success=1
# Parse out optional -c or -n arguments
while getopts "c:ngh" o; do
case "${o}" in
@ -71,7 +74,7 @@ need_file(){
for f in $*; do
if [ ! -e $f ]; then
echo "ERROR: $f does not exist."
exit -1
exit 1
fi
done
}
@ -84,6 +87,7 @@ need_file ${rpms_from_centos_repo}
need_file ${other_downloads}
need_file tarball-dl.lst mvn-artifacts.lst
#download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget"
echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..."
@ -97,43 +101,90 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
fi
logfile="log_download_3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ]; then
echo "ERROR: something wrong with downloading, please check the log!!"
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo ""
success=0
fi
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
logfile="log_download_centos3rdparties_L1.txt"
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ];then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_3rd_parties}."
echo " Please check the log at $(pwd)/logs/$logfile !"
echo ""
success=0
fi
if [ ${use_system_yum_conf} -eq 1 ]; then
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
\rm -f $REPO_DIR/StarlingX_3rd*.repo
fi
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
logfile="log_download_centos_L1.txt"
$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
$rpm_downloader ${rpms_from_centos_repo} L1 |& tee ./logs/$logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -ne 0 ]; then
K1_logfile="log_download_rpms_from_centos_K1.txt"
if [ $retcode -ne 1 ]; then
# K1 step not needed. Clear any K1 logs from previous download attempts.
$rpm_downloader -x ./output/centos_rpms_missing_L1.txt K1 |& tee ./logs/$K1_logfile
fi
if [ $retcode -eq 0 ]; then
echo "finish 1st round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 1st round of RPM downloading with missing files!"
if [ -e "./output/centos_rpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1`
echo "start 2nd round of downloading Binary RPMs with K1 match criteria..."
$rpm_downloader ./output/centos_rpms_missing_L1.txt K1 centos |& tee ./logs/$K1_logfile
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ]; then
echo "finish 2nd round of RPM downloading successfully!"
elif [ $retcode -eq 1 ]; then
echo "finish 2nd round of RPM downloading with missing files!"
if [ -e "./output/rpms_missing_K1.txt" ]; then
echo "WARNING: missing RPMs listed in ./output/centos_rpms_missing_K1.txt !"
fi
fi
# Remove files found by K1 download from centos_rpms_missing_L1.txt to prevent
# false reporting of missing files.
grep -v -x -F -f ./output/centos_rpms_found_K1.txt ./output/centos_rpms_missing_L1.txt > ./output/centos_rpms_missing_L1.tmp
mv -f ./output/centos_rpms_missing_L1.tmp ./output/centos_rpms_missing_L1.txt
missing_num=`wc -l ./output/centos_rpms_missing_K1.txt | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------"
echo "ERROR: -------RPMs missing: $missing_num ---------------"
retcode=1
fi
fi
if [ -e "./output/centos_srpms_missing_L1.txt" ]; then
missing_num=`wc -l ./output/centos_srpms_missing_L1.txt | cut -d " " -f1-1`
if [ "$missing_num" != "0" ];then
echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------"
echo "ERROR: --------- SRPMs missing: $missing_num ---------------"
retcode=1
fi
fi
else
echo "finish 1st round with failures!"
fi
if [ $retcode -ne 0 ]; then
echo "ERROR: Something wrong with downloading files listed in ${rpms_from_centos_repo}."
echo " Please check the logs at $(pwd)/logs/$logfile"
echo " and $(pwd)/logs/$K1_logfile !"
echo ""
success=0
fi
## verify all RPMs SRPMs we download for the GPG keys
@ -147,7 +198,7 @@ line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
let total_line=$line1+$line2+$line3
echo "We expect to download $total_line RPMs."
echo "We expected to download $total_line RPMs."
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
echo "There are $num_of_downloaded_rpms RPMs in output directory."
if [ "$total_line" != "$num_of_downloaded_rpms" ]; then
@ -162,16 +213,33 @@ fi
echo "step #3: start downloading other files ..."
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt
${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ |& tee ./logs/log_download_other_files_centos.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #3: done successfully"
else
echo "step #3: finished with errors"
echo "ERROR: Something wrong with downloading from ${other_downloads}."
echo " Please check the log at $(pwd)/logs/log_download_other_files_centos.txt !"
echo ""
success=0
fi
# StarlingX requires a group of source code pakages, in this section
# they will be downloaded.
echo "step #4: start downloading tarball compressed files"
${tarball_downloader} ${tarball_downloader_extra_args}
${tarball_downloader} ${tarball_downloader_extra_args} |& tee ./logs/log_download_tarballs.txt
retcode=${PIPESTATUS[0]}
if [ $retcode -eq 0 ];then
echo "step #4: done successfully"
else
echo "step #4: finished with errors"
echo "ERROR: Something wrong with downloading tarballs."
echo " Please check the log at $(pwd)/logs/log_download_tarballs.txt !"
echo ""
success=0
fi
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
echo "on them, the workable images for StarlingX could be generated by"
@ -179,3 +247,12 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
echo ""
if [ $success -ne 1 ]; then
echo "Warning: Not all download steps succeeded. You are likely missing files."
exit 1
fi
echo "Success"
exit 0