Merge "Change lists names and downloading scripts"

centos-mirror-tools/dl_rpms.sh

@@ -6,7 +6,7 @@
 # this script was originated by Brian Avery, and later updated by Yong Hu
 
 usage() {
-    echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> <from_where>"
+    echo "$0 [-n] [-c <yum.conf>] <rpms_list> <match_level> "
     echo ""
     echo "Options:"
     echo "  -n: Do not use sudo when performing operations"
@@ -19,11 +19,16 @@ usage() {
     echo "        using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm"
     echo "    L3: use name:"
     echo "        using vim to search vim-7.4.160-2.el7.src.rpm"
-    echo "  from_where: where to download the RPMs: 'centos'from CentOS Repos,"
-    echo "              otherwise from 3rd-party websets"
     echo ""
 }
 
+get_from() {
+    list=$1
+    base=$(basename $list .lst)
+    from=$(echo $base | cut -d'_' -f2-2)
+    echo $from
+}
+
 # By default, we use "sudo" and we don't use a local yum.conf. These can
 # be overridden via flags.
 SUDOCMD="sudo -E"
@@ -53,7 +58,7 @@ while getopts "c:nh" o; do
 done
 shift $((OPTIND-1))
 
-if [ $# -lt 3 ]; then
+if [ $# -lt 2 ]; then
     usage
     exit -1
 fi
@@ -72,7 +77,6 @@ if [ ! -z "$2" -a "$2" != " " ];then
     match_level=$2
 fi
 
-from=$3
 
 timestamp=$(date +%F_%H%M)
 echo $timestamp
@@ -83,6 +87,8 @@ mkdir -p $MDIR_SRC
 MDIR_BIN=$DESTDIR/stx-r1/CentOS/pike/Binary
 mkdir -p $MDIR_BIN
 
+
+from=$(get_from $rpms_list)
 FAIL_MOVE_SRPMS="$DESTDIR/${from}_srpms_fail_move_${match_level}.txt"
 FOUND_SRPMS="$DESTDIR/${from}_srpms_found_${match_level}.txt"
 MISSING_SRPMS="$DESTDIR/${from}_srpms_missing_${match_level}.txt"
@@ -103,18 +109,21 @@ cat /dev/null > $FOUND_RPMS
 cat /dev/null > $MISSING_RPMS
 cat /dev/null > $URL_RPMS
 
-#function to download different type of RPMs in different ways
+# Function to download different types of RPMs in different ways
 download () {
-    _list=$1
+    _file=$1
     _level=$2
-    _from=$3
-    _type=$4
-
+    _list=$(cat $_file)
+    _from=$(get_from $_file)
     echo "now the rpm will come from: $_from"
     for ff in $_list; do
-        ## download RPM from CentOS repos
-        if [ "$_from" == "centos" -o "$_from" == "3rd-centos" ]; then
+        _type=$(echo $ff | rev | cut -d'.' -f2-2 | rev)
+
+        # Decide if the list will be downloaded using yumdownloader or wget
+        if [[ $ff != *"#"* ]]; then
             rpm_name=$ff
+
+            # Cut the rpm name for the specified level (L1, L2 or L3)
             if [ $_level == "L1" ]; then
                 SFILE=`echo $rpm_name | rev | cut -d'.' -f3- | rev`
             elif [ $match_level == "L2" ];then
@@ -123,6 +132,7 @@ download () {
                 SFILE=`echo $rpm_name | rev | cut -d'-' -f3- | rev`
             fi
             echo " ------ using $SFILE to search $rpm_name ------"
+            # Yumdownloader with the appropriate flag for src, noarch or x86_64
             if [ "$_type" == "src" ];then
                 download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE"
                 download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE"
@@ -131,12 +141,15 @@ download () {
                 download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64"
             fi
         else
+            # Buid wget command
             rpm_name=`echo $ff | cut -d"#" -f1-1`
             rpm_url=`echo $ff | cut -d"#" -f2-2`
             download_cmd="wget $rpm_url"
             SFILE=$rpm_name
         fi
+
         echo "--> run: $download_cmd"
+        # Put the RPM in the Binary or Source directory
         if [ "$_type" == "src" ]; then
             if [ ! -e $MDIR_SRC/$rpm_name ]; then
                 echo "Looking for $rpm_name"
@@ -163,7 +176,7 @@ download () {
                 echo "Looking for $rpm_name..."
                 if $download_cmd ; then
                     # Success!   Record download URL.
-                    # Use 'sort --unique' because sometimes 
+                    # Use 'sort --unique' because sometimes
                     # yumdownloader reports the url twice
                     $download_url_cmd | sort --unique >> $URL_RPMS
 
@@ -184,28 +197,13 @@ download () {
     done
 }
 
-# prime the cache
+# Prime the cache
 ${SUDOCMD} yum ${YUMCONFOPT} makecache
 
-#go to download *.noarch.rpm files
-noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"`
-if [ ! -z "$noarch_rpms" ];then
-    echo "--> start searching noarch RPMs ....."
-    download "$noarch_rpms" $match_level $from "noarch"
-fi
-
-#go to download *.x86_64.rpm files
-x86_64_rpms=`echo "$(cat $rpms_list | grep '.x86_64.rpm')"`
-if [ ! -z "$x86_64_rpms" ];then
-    echo "--> start searching x86_64 RPMs ....."
-    download "$x86_64_rpms" $match_level $from "x86_64"
-fi
-
-#go to download *.src.rpm files
-src_rpms=`echo "$(cat $rpms_list | grep '.src.rpm')"`
-if [ ! -z "$src_rpms" ];then
-    echo "--> start searching source RPMs ....."
-    download "$src_rpms" $match_level $from "src"
+# Download files
+if [ -s "$rpms_list" ];then
+    echo "--> start searching "$rpms_list
+    download $rpms_list $match_level
 fi
 
 echo "done!!"

centos-mirror-tools/download_mirror.sh

@@ -26,9 +26,9 @@ rpm_downloader_extra_args=""
 tarball_downloader_extra_args=""
 
 # lst files to use as input
-rpms_from_3rd_parties="./rpms_from_3rd_parties.lst"
-rpms_from_centos_repo="./rpms_from_centos_repo.lst"
-rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst"
+rpms_from_3rd_parties="./rpms_3rdparties.lst"
+rpms_from_centos_repo="./rpms_centos.lst"
+rpms_from_centos_3rd_parties="./rpms_centos3rdparties.lst"
 other_downloads="./other_downloads.lst"
 
 # Parse out optional -c or -n arguments
@@ -96,14 +96,17 @@ if [ ${use_system_yum_conf} -ne 0 ]; then
     fi
 fi
 
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt
+logfile="log_download_3rdparties_L1.txt"
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 | tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
-if [ $retcode -ne 0 ];then
+if [ $retcode -ne 0 ]; then
     echo "ERROR: something wrong with downloading, please check the log!!"
 fi
 
 # download RPMs/SRPMs from 3rd_party repos by "yumdownloader"
-$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt
+logfile="log_download_centos3rdparties_L1.txt"
+$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 | tee ./logs/$logfile
+
 if [ ${use_system_yum_conf} -eq 1 ]; then
     # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo.
     \rm -f $REPO_DIR/StarlingX_3rd*.repo
@@ -111,9 +114,8 @@ fi
 
 echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..."
 #download RPMs/SRPMs from CentOS repos by "yumdownloader"
-
-$rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt
-
+logfile="log_download_centos_L1.txt"
+$rpm_downloader ${rpms_from_centos_repo} L1 | tee ./logs/$logfile
 retcode=${PIPESTATUS[0]}
 if [ $retcode -ne 0 ]; then
     echo "finish 1st round of RPM downloading successfully!"
@@ -144,7 +146,6 @@ find ./output -name "*.i686.rpm" | xargs rm -f
 line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1`
 line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1`
 line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1`
-
 let total_line=$line1+$line2+$line3
 echo "We expect to download $total_line RPMs."
 num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1`
@@ -178,4 +179,3 @@ echo "running \"update-pxe-network-installer\" command after \"build-iso\""
 echo "    - out/stx-r1/CentOS/pike/Binary/LiveOS/squashfs.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/initrd.img"
 echo "    - out/stx-r1/CentOS/pike/Binary/images/pxeboot/vmlinuz"
-