starlingx/tools
Code Issues Proposed changes

Merge "Avoid to produce broken tarballs by checking the mime type."

Browse Source
This commit is contained in:
Zuul 2019-04-18 14:39:09 +00:00 committed by Gerrit Code Review
commit d00e0c3ebf
1 changed files with 57 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
centos-mirror-tools/dl_tarball.sh
View File

@ -121,10 +121,25 @@ if [ ! -d "$logs_dir" ]; then
mkdir "$logs_dir" mkdir "$logs_dir"
fi fi
is_tarball() {
tarball_name="$1"
mime_type=$(file --mime-type -b $tarball_name | cut -d "/" -f 2)
types=("gzip" "x-bzip2" "x-rpm" "x-xz" "x-gzip" "x-tar")
FOUND=1
for t in "${types[@]}"; do
if [ "$mime_type" == "$t" ]; then
FOUND=0
break;
fi
done
return $FOUND
}
# Download function using wget command # Download function using wget command
download_package() { download_package() {
local upstream_url="$1" local tarball_name="$1"
local upstream_url="$2"
local stx_url="" local stx_url=""
local url="" local url=""
local rc=1 local rc=1
@ -149,10 +164,17 @@ download_package() {
if [ $? != 0 ]; then if [ $? != 0 ]; then
echo "Warning: '$url' is broken" echo "Warning: '$url' is broken"
else else
wget -t 5 --wait=15 "$url" wget -q -t 5 --wait=15 -O "$tarball_name" "$url"
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
rc=0 if is_tarball "$tarball_name"; then
break echo "Ok: $download_path"
rc=0
break
else
echo "Warning: File from '$url' is not a tarball"
\rm "$tarball_name"
rc=1
fi
else else
echo "Warning: failed to download '$url'" echo "Warning: failed to download '$url'"
continue continue
@ -240,46 +262,46 @@ for line in $(cat $tarball_file); do
if [[ "$line" =~ ^'!' ]]; then if [[ "$line" =~ ^'!' ]]; then
echo $tarball_name echo $tarball_name
pushd $output_tarball pushd $output_tarball > /dev/null
if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then
download_package $tarball_url download_package "$tarball_name" "$tarball_url"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz tar xf "$tarball_name"
rm "$tarball_name"
mv linux-tpmdd-e6aef06/security/integrity/ $directory_name mv linux-tpmdd-e6aef06/security/integrity/ $directory_name
tar czvf $tarball_name $directory_name tar czvf $tarball_name $directory_name
rm -rf linux-tpmdd-e6aef06 rm -rf linux-tpmdd-e6aef06
rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then
download_package $tarball_url download_package "$tarball_name" "$tarball_url"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
mkdir $directory_name mkdir $directory_name
tar xf $tarball_name --strip-components 1 -C $directory_name tar xf $tarball_name --strip-components 1 -C $directory_name
rm $tarball_name rm $tarball_name
pushd $directory_name pushd $directory_name > /dev/null
rm -rf storage/tokudb rm -rf storage/tokudb
rm ./man/tokuft_logdump.1 ./man/tokuftdump.1 rm ./man/tokuft_logdump.1 ./man/tokuftdump.1
sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt
sed -e s/tokuftdump.1//g -i man/CMakeLists.txt sed -e s/tokuftdump.1//g -i man/CMakeLists.txt
popd popd > /dev/null
tar czvf $tarball_name $directory_name tar czvf $tarball_name $directory_name
rm -rf $directory_name rm -rf $directory_name
popd # pushd $directory_name popd > /dev/null # pushd $directory_name
elif [[ "$tarball_name" = 'MLNX_OFED_SRC-4.5-1.0.1.0.tgz' ]]; then elif [[ "$tarball_name" = 'MLNX_OFED_SRC-4.5-1.0.1.0.tgz' ]]; then
srpm_path="${directory_name}/SRPMS/" srpm_path="${directory_name}/SRPMS/"
download_package "$tarball_url" download_package "$tarball_name" "$tarball_url"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
@ -293,27 +315,27 @@ for line in $(cat $tarball_file); do
rm -rf "$directory_name" rm -rf "$directory_name"
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then
download_package $tarball_url download_package "$tarball_name" "$tarball_url"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then elif [ "$tarball_name" = "tpm-kmod-e6aef069.tar.gz" ]; then
download_package $tarball_url download_package "$tarball_name" "$tarball_url"
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
tar xf e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz tar xf "$tarball_name"
rm "$tarball_name"
mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name mv linux-tpmdd-e6aef06/drivers/char/tpm $directory_name
tar czvf $tarball_name $directory_name tar czvf $tarball_name $directory_name
rm -rf linux-tpmdd-e6aef06 rm -rf linux-tpmdd-e6aef06
rm -rf $directory_name rm -rf $directory_name
rm e6aef069b6e97790cb127d5eeb86ae9ff0b7b0e3.tar.gz
elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then
dest_dir=ibmtpm20tss-tss dest_dir=ibmtpm20tss-tss
for dl_src in $dl_source; do for dl_src in $dl_source; do
@ -344,21 +366,21 @@ for line in $(cat $tarball_file); do
echo "Error: Failed to git clone from '$tarball_url'" echo "Error: Failed to git clone from '$tarball_url'"
echo "$tarball_url" > "$output_log" echo "$tarball_url" > "$output_log"
error_count=$((error_count + 1)) error_count=$((error_count + 1))
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
pushd $dest_dir pushd $dest_dir > /dev/null
branch=$util branch=$util
git checkout $branch git checkout $branch
rm -rf .git rm -rf .git
popd popd > /dev/null
mv ibmtpm20tss-tss $directory_name mv ibmtpm20tss-tss $directory_name
tar czvf $tarball_name $directory_name tar czvf $tarball_name $directory_name
rm -rf $directory_name rm -rf $directory_name
popd # pushd $dest_dir popd > /dev/null # pushd $dest_dir
fi fi
popd # pushd $output_tarball popd > /dev/null # pushd $output_tarball
continue continue
fi fi
@ -381,11 +403,16 @@ for line in $(cat $tarball_file); do
;; ;;
esac esac
download_cmd="wget -t 5 --wait=15 $url -O $download_path" download_cmd="wget -q -t 5 --wait=15 $url -O $download_path"
if $download_cmd ; then if $download_cmd ; then
if ! is_tarball "$download_path"; then
echo "Warning: file from $url is not a tarball."
\rm "$download_path"
continue
fi
echo "Ok: $download_path" echo "Ok: $download_path"
pushd $download_directory pushd $download_directory > /dev/null
directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/") directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/")
if [ "$directory_name" != "$directory_name_original" ]; then if [ "$directory_name" != "$directory_name_original" ]; then
mkdir -p $directory_name mkdir -p $directory_name
@ -393,7 +420,7 @@ for line in $(cat $tarball_file); do
tar -czf $tarball_name $directory_name tar -czf $tarball_name $directory_name
rm -r $directory_name rm -r $directory_name
fi fi
popd popd > /dev/null
break break
else else
echo "Warning: Failed to download $url" 1>&2 echo "Warning: Failed to download $url" 1>&2