Avoid to produce broken tarballs by checking the mime type.
The tarball downloader uses wget to retrieve content from the urls. In some cases, this url doesn't point directly to a file in the server. For cases like sourceforge, the url points to an API that will deliver the requested file. This is problematic for wget because still gets a 200 HTTP reponse even if the is not there, and then downloads a text file with the content of the requested page. This fix proposes to check the mime type of the downloaded file and if the file is not one of the used types the it will delete it. Closes-Bug: 1795739 Change-Id: Ia9286ca094389d8d5196fce8ccda8a38cc754075 Signed-off-by: Erich Cordoba <erich.cordoba.malibran@intel.com>
This commit is contained in:
parent
83e1533e47
commit
43574df058
@ -121,10 +121,25 @@ if [ ! -d "$logs_dir" ]; then
|
||||
mkdir "$logs_dir"
|
||||
fi
|
||||
|
||||
is_tarball() {
|
||||
tarball_name="$1"
|
||||
mime_type=$(file --mime-type -b $tarball_name | cut -d "/" -f 2)
|
||||
types=("gzip" "x-bzip2" "x-rpm" "x-xz" "x-gzip" "x-tar")
|
||||
FOUND=1
|
||||
for t in "${types[@]}"; do
|
||||
if [ "$mime_type" == "$t" ]; then
|
||||
FOUND=0
|
||||
break;
|
||||
fi
|
||||
done
|
||||
return $FOUND
|
||||
}
|
||||
|
||||
# Download function using wget command
|
||||
|
||||
download_package() {
|
||||
local upstream_url="$1"
|
||||
local tarball_name="$1"
|
||||
local upstream_url="$2"
|
||||
local stx_url=""
|
||||
local url=""
|
||||
local rc=1
|
||||
@ -149,10 +164,17 @@ download_package() {
|
||||
if [ $? != 0 ]; then
|
||||
echo "Warning: '$url' is broken"
|
||||
else
|
||||
wget -t 5 --wait=15 "$url"
|
||||
wget -q -t 5 --wait=15 -O "$tarball_name" "$url"
|
||||
if [ $? -eq 0 ]; then
|
||||
rc=0
|
||||
break
|
||||
if is_tarball "$tarball_name"; then
|
||||
echo "Ok: $download_path"
|
||||
rc=0
|
||||
break
|
||||
else
|
||||
echo "Warning: File from '$url' is not a tarball"
|
||||
\rm "$tarball_name"
|
||||
rc=1
|
||||
fi
|
||||
else
|
||||
echo "Warning: failed to download '$url'"
|
||||
continue
|
||||
@ -240,46 +262,46 @@ for line in $(cat $tarball_file); do
|
||||
|
||||
if [[ "$line" =~ ^'!' ]]; then
|
||||
echo $tarball_name
|
||||
pushd $output_tarball
|
||||
pushd $output_tarball > /dev/null
|
||||
if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
download_package "$tarball_name" "$tarball_url"
|
||||
if [ $? -ne 0 ]; then
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
|
||||
tar xf "$tarball_name"
|
||||
rm "$tarball_name"
|
||||
mv linux-tpmdd-e6aef06/security/integrity/ $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf linux-tpmdd-e6aef06
|
||||
rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
|
||||
elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
download_package "$tarball_name" "$tarball_url"
|
||||
if [ $? -ne 0 ]; then
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir $directory_name
|
||||
tar xf $tarball_name --strip-components 1 -C $directory_name
|
||||
rm $tarball_name
|
||||
pushd $directory_name
|
||||
pushd $directory_name > /dev/null
|
||||
rm -rf storage/tokudb
|
||||
rm ./man/tokuft_logdump.1 ./man/tokuftdump.1
|
||||
sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt
|
||||
sed -e s/tokuftdump.1//g -i man/CMakeLists.txt
|
||||
popd
|
||||
popd > /dev/null
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf $directory_name
|
||||
popd # pushd $directory_name
|
||||
popd > /dev/null # pushd $directory_name
|
||||
elif [[ "$tarball_name" = 'MLNX_OFED_SRC-4.5-1.0.1.0.tgz' ]]; then
|
||||
srpm_path="${directory_name}/SRPMS/"
|
||||
download_package "$tarball_url"
|
||||
download_package "$tarball_name" "$tarball_url"
|
||||
if [ $? -ne 0 ]; then
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
@ -293,27 +315,27 @@ for line in $(cat $tarball_file); do
|
||||
|
||||
rm -rf "$directory_name"
|
||||
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
download_package "$tarball_name" "$tarball_url"
|
||||
if [ $? -ne 0 ]; then
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then
|
||||
download_package $tarball_url
|
||||
download_package "$tarball_name" "$tarball_url"
|
||||
if [ $? -ne 0 ]; then
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
|
||||
tar xf "$tarball_name"
|
||||
rm "$tarball_name"
|
||||
mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf linux-tpmdd-e6aef06
|
||||
rm -rf $directory_name
|
||||
rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
|
||||
elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then
|
||||
dest_dir=ibmtpm20tss-tss
|
||||
for dl_src in $dl_source; do
|
||||
@ -344,21 +366,21 @@ for line in $(cat $tarball_file); do
|
||||
echo "Error: Failed to git clone from '$tarball_url'"
|
||||
echo "$tarball_url" > "$output_log"
|
||||
error_count=$((error_count + 1))
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
pushd $dest_dir
|
||||
pushd $dest_dir > /dev/null
|
||||
branch=$util
|
||||
git checkout $branch
|
||||
rm -rf .git
|
||||
popd
|
||||
popd > /dev/null
|
||||
mv ibmtpm20tss-tss $directory_name
|
||||
tar czvf $tarball_name $directory_name
|
||||
rm -rf $directory_name
|
||||
popd # pushd $dest_dir
|
||||
popd > /dev/null # pushd $dest_dir
|
||||
fi
|
||||
popd # pushd $output_tarball
|
||||
popd > /dev/null # pushd $output_tarball
|
||||
continue
|
||||
fi
|
||||
|
||||
@ -381,11 +403,16 @@ for line in $(cat $tarball_file); do
|
||||
;;
|
||||
esac
|
||||
|
||||
download_cmd="wget -t 5 --wait=15 $url -O $download_path"
|
||||
download_cmd="wget -q -t 5 --wait=15 $url -O $download_path"
|
||||
|
||||
if $download_cmd ; then
|
||||
if ! is_tarball "$download_path"; then
|
||||
echo "Warning: file from $url is not a tarball."
|
||||
\rm "$download_path"
|
||||
continue
|
||||
fi
|
||||
echo "Ok: $download_path"
|
||||
pushd $download_directory
|
||||
pushd $download_directory > /dev/null
|
||||
directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/")
|
||||
if [ "$directory_name" != "$directory_name_original" ]; then
|
||||
mkdir -p $directory_name
|
||||
@ -393,7 +420,7 @@ for line in $(cat $tarball_file); do
|
||||
tar -czf $tarball_name $directory_name
|
||||
rm -r $directory_name
|
||||
fi
|
||||
popd
|
||||
popd > /dev/null
|
||||
break
|
||||
else
|
||||
echo "Warning: Failed to download $url" 1>&2
|
||||
|
Loading…
Reference in New Issue
Block a user