Merge "Change lists names and downloading scripts"
This commit is contained in:
commit
219a0de972
@ -6,7 +6,7 @@
|
|||||||
# this script was originated by Brian Avery, and later updated by Yong Hu
|
# this script was originated by Brian Avery, and later updated by Yong Hu
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> <from_where>"
|
echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
|
||||||
echo ""
|
echo ""
|
||||||
echo "Options:"
|
echo "Options:"
|
||||||
echo " -n: Do not use sudo when performing operations"
|
echo " -n: Do not use sudo when performing operations"
|
||||||
@ -19,11 +19,16 @@ usage() {
|
|||||||
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
|
echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
|
||||||
echo " L3: use name:"
|
echo " L3: use name:"
|
||||||
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
|
echo " using vim to search vim-7.4.160-2.el7.src.rpm"
|
||||||
echo " from_where: where to download the RPMs: 'centos'from CentOS Repos,"
|
|
||||||
echo " otherwise from 3rd-party websets"
|
|
||||||
echo ""
|
echo ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
get_from() {
|
||||||
|
list=$1
|
||||||
|
base=$(basename $list .lst)
|
||||||
|
from=$(echo $base | cut -d'_' -f2-2)
|
||||||
|
echo $from
|
||||||
|
}
|
||||||
|
|
||||||
# By default, we use "sudo" and we don't use a local yum.conf. These can
|
# By default, we use "sudo" and we don't use a local yum.conf. These can
|
||||||
# be overridden via flags.
|
# be overridden via flags.
|
||||||
SUDOCMD="sudo -E"
|
SUDOCMD="sudo -E"
|
||||||
@ -53,7 +58,7 @@ while getopts "c:nh" o; do
|
|||||||
done
|
done
|
||||||
shift $((OPTIND-1))
|
shift $((OPTIND-1))
|
||||||
|
|
||||||
if [ $# -lt 3 ]; then
|
if [ $# -lt 2 ]; then
|
||||||
usage
|
usage
|
||||||
exit -1
|
exit -1
|
||||||
fi
|
fi
|
||||||
@ -72,7 +77,6 @@ if [ ! -z "$2" -a "$2" != " " ];then
|
|||||||
match_level=$2
|
match_level=$2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
from=$3
|
|
||||||
|
|
||||||
timestamp=$(date +%F_%H%M)
|
timestamp=$(date +%F_%H%M)
|
||||||
echo $timestamp
|
echo $timestamp
|
||||||
@ -83,6 +87,8 @@ mkdir -p $MDIR_SRC
|
|||||||
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
|
MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
|
||||||
mkdir -p $MDIR_BIN
|
mkdir -p $MDIR_BIN
|
||||||
|
|
||||||
|
|
||||||
|
from=$(get_from $rpms_list)
|
||||||
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
|
FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
|
||||||
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
|
FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
|
||||||
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
|
MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
|
||||||
@ -103,18 +109,21 @@ cat /dev/null > $FOUND_RPMS
|
|||||||
cat /dev/null > $MISSING_RPMS
|
cat /dev/null > $MISSING_RPMS
|
||||||
cat /dev/null > $URL_RPMS
|
cat /dev/null > $URL_RPMS
|
||||||
|
|
||||||
#function to download different type of RPMs in different ways
|
# Function to download different types of RPMs in different ways
|
||||||
download () {
|
download () {
|
||||||
_list=$1
|
_file=$1
|
||||||
_level=$2
|
_level=$2
|
||||||
_from=$3
|
_list=$(cat $_file)
|
||||||
_type=$4
|
_from=$(get_from $_file)
|
||||||
|
|
||||||
echo "now the rpm will come from: $_from"
|
echo "now the rpm will come from: $_from"
|
||||||
for ff in $_list; do
|
for ff in $_list; do
|
||||||
## download RPM from CentOS repos
|
_type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
|
||||||
if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then
|
|
||||||
|
# Decide if the list will be downloaded using yumdownloader or wget
|
||||||
|
if [[ $ff != *"#"* ]]; then
|
||||||
rpm_name=$ff
|
rpm_name=$ff
|
||||||
|
|
||||||
|
# Cut the rpm name for the specified level (L1, L2 or L3)
|
||||||
if [ $_level == "L1" ]; then
|
if [ $_level == "L1" ]; then
|
||||||
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
|
SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
|
||||||
elif [ $match_level == "L2" ];then
|
elif [ $match_level == "L2" ];then
|
||||||
@ -123,6 +132,7 @@ download () {
|
|||||||
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
|
SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
|
||||||
fi
|
fi
|
||||||
echo " ------ using $SFILE to search $rpm_name ------"
|
echo " ------ using $SFILE to search $rpm_name ------"
|
||||||
|
# Yumdownloader with the appropriate flag for src, noarch or x86_64
|
||||||
if [ "$_type" == "src" ];then
|
if [ "$_type" == "src" ];then
|
||||||
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
|
download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
|
||||||
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
|
download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
|
||||||
@ -131,12 +141,15 @@ download () {
|
|||||||
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
|
download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
|
# Buid wget command
|
||||||
rpm_name=`echo $ff | cut -d"#" -f1-1`
|
rpm_name=`echo $ff | cut -d"#" -f1-1`
|
||||||
rpm_url=`echo $ff | cut -d"#" -f2-2`
|
rpm_url=`echo $ff | cut -d"#" -f2-2`
|
||||||
download_cmd="wget $rpm_url"
|
download_cmd="wget $rpm_url"
|
||||||
SFILE=$rpm_name
|
SFILE=$rpm_name
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "--> run: $download_cmd"
|
echo "--> run: $download_cmd"
|
||||||
|
# Put the RPM in the Binary or Source directory
|
||||||
if [ "$_type" == "src" ]; then
|
if [ "$_type" == "src" ]; then
|
||||||
if [ ! -e $MDIR_SRC/$rpm_name ]; then
|
if [ ! -e $MDIR_SRC/$rpm_name ]; then
|
||||||
echo "Looking for $rpm_name"
|
echo "Looking for $rpm_name"
|
||||||
@ -163,7 +176,7 @@ download () {
|
|||||||
echo "Looking for $rpm_name..."
|
echo "Looking for $rpm_name..."
|
||||||
if $download_cmd ; then
|
if $download_cmd ; then
|
||||||
# Success! Record download URL.
|
# Success! Record download URL.
|
||||||
# Use 'sort --unique' because sometimes
|
# Use 'sort --unique' because sometimes
|
||||||
# yumdownloader reports the url twice
|
# yumdownloader reports the url twice
|
||||||
$download_url_cmd | sort --unique >> $URL_RPMS
|
$download_url_cmd | sort --unique >> $URL_RPMS
|
||||||
|
|
||||||
@ -184,28 +197,13 @@ download () {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# prime the cache
|
# Prime the cache
|
||||||
${SUDOCMD} yum ${YUMCONFOPT} makecache
|
${SUDOCMD} yum ${YUMCONFOPT} makecache
|
||||||
|
|
||||||
#go to download *.noarch.rpm files
|
# Download files
|
||||||
noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"`
|
if [ -s "$rpms_list" ];then
|
||||||
if [ ! -z "$noarch_rpms" ];then
|
echo "--> start searching "$rpms_list
|
||||||
echo "--> start searching noarch RPMs ....."
|
download $rpms_list $match_level
|
||||||
download "$noarch_rpms" $match_level $from "noarch"
|
|
||||||
fi
|
|
||||||
|
|
||||||
#go to download *.x86_64.rpm files
|
|
||||||
x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"`
|
|
||||||
if [ ! -z "$x86_64_rpms" ];then
|
|
||||||
echo "--> start searching x86_64 RPMs ....."
|
|
||||||
download "$x86_64_rpms" $match_level $from "x86_64"
|
|
||||||
fi
|
|
||||||
|
|
||||||
#go to download *.src.rpm files
|
|
||||||
src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"`
|
|
||||||
if [ ! -z "$src_rpms" ];then
|
|
||||||
echo "--> start searching source RPMs ....."
|
|
||||||
download "$src_rpms" $match_level $from "src"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "done!!"
|
echo "done!!"
|
||||||
|
@ -26,9 +26,9 @@ rpm_downloader_extra_args=""
|
|||||||
tarball_downloader_extra_args=""
|
tarball_downloader_extra_args=""
|
||||||
|
|
||||||
# lst files to use as input
|
# lst files to use as input
|
||||||
rpms_from_3rd_parties="./rpms_from_3rd_parties.lst"
|
rpms_from_3rd_parties="./rpms_3rdparties.lst"
|
||||||
rpms_from_centos_repo="./rpms_from_centos_repo.lst"
|
rpms_from_centos_repo="./rpms_centos.lst"
|
||||||
rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst"
|
rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
|
||||||
other_downloads="./other_downloads.lst"
|
other_downloads="./other_downloads.lst"
|
||||||
|
|
||||||
# Parse out optional -c or -n arguments
|
# Parse out optional -c or -n arguments
|
||||||
@ -96,14 +96,17 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
|
logfile="log_download_3rdparties_L1.txt"
|
||||||
|
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
|
||||||
retcode=${PIPESTATUS[0]}
|
retcode=${PIPESTATUS[0]}
|
||||||
if [ $retcode -ne 0 ];then
|
if [ $retcode -ne 0 ]; then
|
||||||
echo "ERROR: something wrong with downloading, please check the log!!"
|
echo "ERROR: something wrong with downloading, please check the log!!"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
|
# download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
|
||||||
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
|
logfile="log_download_centos3rdparties_L1.txt"
|
||||||
|
$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
|
||||||
|
|
||||||
if [ ${use_system_yum_conf} -eq 1 ]; then
|
if [ ${use_system_yum_conf} -eq 1 ]; then
|
||||||
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
|
# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
|
||||||
\rm -f $REPO_DIR/StarlingX_3rd*.repo
|
\rm -f $REPO_DIR/StarlingX_3rd*.repo
|
||||||
@ -111,9 +114,8 @@ fi
|
|||||||
|
|
||||||
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
|
echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
|
||||||
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
|
#download RPMs/SRPMs from CentOS repos by "yumdownloader"
|
||||||
|
logfile="log_download_centos_L1.txt"
|
||||||
$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt
|
$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
|
||||||
|
|
||||||
retcode=${PIPESTATUS[0]}
|
retcode=${PIPESTATUS[0]}
|
||||||
if [ $retcode -ne 0 ]; then
|
if [ $retcode -ne 0 ]; then
|
||||||
echo "finish 1st round of RPM downloading successfully!"
|
echo "finish 1st round of RPM downloading successfully!"
|
||||||
@ -144,7 +146,6 @@ find ./output -name "*.i686.rpm" | xargs rm -f
|
|||||||
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
|
line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
|
||||||
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
|
line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
|
||||||
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
|
line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
|
||||||
|
|
||||||
let total_line=$line1+$line2+$line3
|
let total_line=$line1+$line2+$line3
|
||||||
echo "We expect to download $total_line RPMs."
|
echo "We expect to download $total_line RPMs."
|
||||||
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
|
num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
|
||||||
@ -178,4 +179,3 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
|
|||||||
echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
|
echo " - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
|
||||||
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
|
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
|
||||||
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
|
echo " - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user