From 43574df05869f947d52283405f28b95fd253fbff Mon Sep 17 00:00:00 2001 From: Erich Cordoba Date: Tue, 16 Apr 2019 16:34:43 -0500 Subject: [PATCH] Avoid to produce broken tarballs by checking the mime type. The tarball downloader uses wget to retrieve content from the urls. In some cases, this url doesn't point directly to a file in the server. For cases like sourceforge, the url points to an API that will deliver the requested file. This is problematic for wget because still gets a 200 HTTP reponse even if the is not there, and then downloads a text file with the content of the requested page. This fix proposes to check the mime type of the downloaded file and if the file is not one of the used types the it will delete it. Closes-Bug: 1795739 Change-Id: Ia9286ca094389d8d5196fce8ccda8a38cc754075 Signed-off-by: Erich Cordoba --- centos-mirror-tools/dl_tarball.sh | 87 ++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index 50039dbb..92a1d798 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -121,10 +121,25 @@ if [ ! -d "$logs_dir" ]; then mkdir "$logs_dir" fi +is_tarball() { + tarball_name="$1" + mime_type=$(file --mime-type -b $tarball_name | cut -d "/" -f 2) + types=("gzip" "x-bzip2" "x-rpm" "x-xz" "x-gzip" "x-tar") + FOUND=1 + for t in "${types[@]}"; do + if [ "$mime_type" == "$t" ]; then + FOUND=0 + break; + fi + done + return $FOUND +} + # Download function using wget command download_package() { - local upstream_url="$1" + local tarball_name="$1" + local upstream_url="$2" local stx_url="" local url="" local rc=1 @@ -149,10 +164,17 @@ download_package() { if [ $? != 0 ]; then echo "Warning: '$url' is broken" else - wget -t 5 --wait=15 "$url" + wget -q -t 5 --wait=15 -O "$tarball_name" "$url" if [ $? -eq 0 ]; then - rc=0 - break + if is_tarball "$tarball_name"; then + echo "Ok: $download_path" + rc=0 + break + else + echo "Warning: File from '$url' is not a tarball" + \rm "$tarball_name" + rc=1 + fi else echo "Warning: failed to download '$url'" continue @@ -240,46 +262,46 @@ for line in $(cat $tarball_file); do if [[ "$line" =~ ^'!' ]]; then echo $tarball_name - pushd $output_tarball + pushd $output_tarball > /dev/null if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then - download_package $tarball_url + download_package "$tarball_name" "$tarball_url" if [ $? -ne 0 ]; then error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi - tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz + tar xf "$tarball_name" + rm "$tarball_name" mv linux-tpmdd-e6aef06/security/integrity/ $directory_name tar czvf $tarball_name $directory_name rm -rf linux-tpmdd-e6aef06 - rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then - download_package $tarball_url + download_package "$tarball_name" "$tarball_url" if [ $? -ne 0 ]; then error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi mkdir $directory_name tar xf $tarball_name --strip-components 1 -C $directory_name rm $tarball_name - pushd $directory_name + pushd $directory_name > /dev/null rm -rf storage/tokudb rm ./man/tokuft_logdump.1 ./man/tokuftdump.1 sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt sed -e s/tokuftdump.1//g -i man/CMakeLists.txt - popd + popd > /dev/null tar czvf $tarball_name $directory_name rm -rf $directory_name - popd # pushd $directory_name + popd > /dev/null # pushd $directory_name elif [[ "$tarball_name" = 'MLNX_OFED_SRC-4.5-1.0.1.0.tgz' ]]; then srpm_path="${directory_name}/SRPMS/" - download_package "$tarball_url" + download_package "$tarball_name" "$tarball_url" if [ $? -ne 0 ]; then error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi @@ -293,27 +315,27 @@ for line in $(cat $tarball_file); do rm -rf "$directory_name" elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then - download_package $tarball_url + download_package "$tarball_name" "$tarball_url" if [ $? -ne 0 ]; then error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then - download_package $tarball_url + download_package "$tarball_name" "$tarball_url" if [ $? -ne 0 ]; then error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi - tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz + tar xf "$tarball_name" + rm "$tarball_name" mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name tar czvf $tarball_name $directory_name rm -rf linux-tpmdd-e6aef06 rm -rf $directory_name - rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then dest_dir=ibmtpm20tss-tss for dl_src in $dl_source; do @@ -344,21 +366,21 @@ for line in $(cat $tarball_file); do echo "Error: Failed to git clone from '$tarball_url'" echo "$tarball_url" > "$output_log" error_count=$((error_count + 1)) - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi - pushd $dest_dir + pushd $dest_dir > /dev/null branch=$util git checkout $branch rm -rf .git - popd + popd > /dev/null mv ibmtpm20tss-tss $directory_name tar czvf $tarball_name $directory_name rm -rf $directory_name - popd # pushd $dest_dir + popd > /dev/null # pushd $dest_dir fi - popd # pushd $output_tarball + popd > /dev/null # pushd $output_tarball continue fi @@ -381,11 +403,16 @@ for line in $(cat $tarball_file); do ;; esac - download_cmd="wget -t 5 --wait=15 $url -O $download_path" + download_cmd="wget -q -t 5 --wait=15 $url -O $download_path" if $download_cmd ; then + if ! is_tarball "$download_path"; then + echo "Warning: file from $url is not a tarball." + \rm "$download_path" + continue + fi echo "Ok: $download_path" - pushd $download_directory + pushd $download_directory > /dev/null directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/") if [ "$directory_name" != "$directory_name_original" ]; then mkdir -p $directory_name @@ -393,7 +420,7 @@ for line in $(cat $tarball_file); do tar -czf $tarball_name $directory_name rm -r $directory_name fi - popd + popd > /dev/null break else echo "Warning: Failed to download $url" 1>&2