starlingx/tools
Code Issues Proposed changes

Auto update CENGN yum.repos.d

Browse Source 
Keep the CENGN copy of yum.repos.d and rpm-gpg-keys updated
relative to the master copies found in stx-tools.

It is not a blind copy.  Rather we add new repo files and
repo id's but never delete old ones.

If a pre-existing repo id has it's url change,
    e.g. mirror.centos.org -> vault.centos.org
we keep a record of the old url under a modified repo id.

This update also relocates a number of common functions
into utility scripts that can be sourced.

Story: 2003906
Task: 26785
Change-Id: I2b5ef46ad7cfe2a3575a3d3d7c91a9511da5ee3c
Signed-off-by: Scott Little <scott.little@windriver.com>
This commit is contained in:
Scott Little 2019-03-15 14:09:57 -04:00
parent 46bbf79317
commit e3c49d3389
6 changed files with 774 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh → centos-mirror-tools/stx_mirror_scripts/dl_sync.sh
View File

@ -3,9 +3,10 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Daily update script for mirror.starlingx.cengn.ca covering
# Update script for mirror.starlingx.cengn.ca covering
# tarballs and other files not downloaded from a yum repository.
# This script was originated by Scott Little.
# The list of files to download are pulled from the .lst files
# found in the stx-tools repo.
#
# IMPORTANT: This script is only to be run on the StarlingX mirror.
# It is not for use by the general StarlinX developer.
@ -15,24 +16,25 @@
DAILY_DL_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
if [ -f "$DAILY_DL_SYNC_DIR/url_utils.sh" ]; then
source "$DAILY_DL_SYNC_DIR/url_utils.sh"
elif [ -f "$DAILY_DL_SYNC_DIR/../url_utils.sh" ]; then
source "$DAILY_DL_SYNC_DIR/../url_utils.sh"
else
echo "Error: Can't find 'url_utils.sh'"
exit 1
fi
LOGFILE=/export/log/daily_dl_sync.log
DOWNLOAD_PATH_ROOT=/export/mirror/centos
STX_TOOLS_BRANCH="master"
STX_TOOLS_BRANCH_ROOT_DIR="$HOME"
STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git"
STX_TOOLS_BRANCH_ROOT_DIR="$HOME/stx-tools"
STX_TOOLS_OS_SUBDIR="centos-mirror-tools"
if [ -f "$DAILY_DL_SYNC_DIR/stx_tool_utils.sh" ]; then
source "$DAILY_DL_SYNC_DIR/stx_tool_utils.sh"
elif [ -f "$DAILY_DL_SYNC_DIR/../stx_tool_utils.sh" ]; then
source "$DAILY_DL_SYNC_DIR/../stx_tool_utils.sh"
else
echo "Error: Can't find 'stx_tool_utils.sh'"
exit 1
fi
usage () {
echo "$0 [-b <branch>] [-d <dir>]"
echo ""
@ -76,151 +78,6 @@ STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH"
STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools"
LST_FILE_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR"
dl_git_from_url () {
local GIT_URL="$1"
local BRANCH="$2"
local DL_DIR="$3"
local DL_ROOT_DIR=""
local SAVE_DIR
local CMD=""
SAVE_DIR="$(pwd)"
if [ "$DL_DIR" == "" ]; then
DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')"
fi
echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'"
DL_ROOT_DIR=$(dirname "$DL_DIR")
if [ ! -d "$DL_DIR" ]; then
if [ ! -d "$DL_ROOT_DIR" ]; then
CMD="mkdir -p '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
CMD="cd '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git clone --bare '$GIT_URL' '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="cd '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git --bare update-server-info"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
if [ -f hooks/post-update.sample ]; then
CMD="mv -f hooks/post-update.sample hooks/post-update"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
fi
CMD="cd '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git fetch"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
cd "$SAVE_DIR"
return 0
}
dl_file_from_url () {
local URL="$1"
local DOWNLOAD_PATH=""
local DOWNLOAD_DIR=""
local PROTOCOL=""
local CMD=""
DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")"
DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")"
PROTOCOL=$(url_protocol $URL)
echo "$PROTOCOL $URL $DOWNLOAD_PATH"
if [ -f "$DOWNLOAD_PATH" ]; then
echo "Already have '$DOWNLOAD_PATH'"
return 0
fi
case "$PROTOCOL" in
https|http)
if [ ! -d "$DOWNLOAD_DIR" ]; then
CMD="mkdir -p '$DOWNLOAD_DIR'"
echo "$CMD"
eval "$CMD"
if [ $? -ne 0 ]; then
echo "Error: $CMD"
return 1
fi
fi
CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
return 1
fi
;;
*)
echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'"
;;
esac
return 0
}
raw_dl_from_rpm_lst () {
local FILE="$1"
@ -238,6 +95,7 @@ raw_dl_from_rpm_lst () {
return $ERR_COUNT
}
raw_dl_from_non_rpm_lst () {
local FILE="$1"
local TAR=""
@ -290,7 +148,7 @@ raw_dl_from_non_rpm_lst () {
;;
git|git_script)
BRANCH="$UTIL"
dl_git_from_url "$URL" "$BRANCH" ""
dl_bare_git_from_url "$URL" ""
if [ $? -ne 0 ]; then
echo "Error: Failed to download '$URL' while processing '$TARBALL_NAME'"
ERR_COUNT=$((ERR_COUNT+1))
@ -314,24 +172,6 @@ raw_dl_from_non_rpm_lst () {
}
stx_tool_clone_or_update () {
local CMD
CMD="mkdir -p '$STX_TOOLS_DL_DIR'"
echo "$CMD"
eval "$CMD"
if [ $? -ne 0 ]; then
echo "Error: $CMD"
return 1
fi
dl_git_from_url "$STX_TOOLS_GIT_URL" "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_DIR"
if [ $? -ne 0 ]; then
echo "Error: Failed to download '$STX_TOOLS_GIT_URL'"
return 1;
fi
return 0
}
if [ -f $LOGFILE ]; then
@ -341,7 +181,7 @@ fi
(
ERR_COUNT=0
stx_tool_clone_or_update
stx_tool_clone_or_update "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_ROOT_DIR"
if [ $? -ne 0 ]; then
echo "Error: Failed to update stx_tools. Can't continue."
exit 1

277
centos-mirror-tools/stx_mirror_scripts/dl_utils.sh Normal file
View File

@ -0,0 +1,277 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Utility function for the download of gits and tarballs.
#
# This script was originated by Scott Little.
#
DL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
if [ -f "$DL_UTILS_DIR/url_utils.sh" ]; then
source "$DL_UTILS_DIR/url_utils.sh"
elif [ -f "$DL_UTILS_DIR/../url_utils.sh" ]; then
source "$DL_UTILS_DIR/../url_utils.sh"
else
echo "Error: Can't find 'url_utils.sh'"
exit 1
fi
DOWNLOAD_PATH_ROOT=${DOWNLOAD_PATH_ROOT:-/export/mirror/centos}
#
# dl_git_from_url <git-url> <branch> <dir>
#
# Download a git from supplied url into directory,
# and checkout desired branch.
#
dl_git_from_url () {
local GIT_URL="$1"
local BRANCH="$2"
local DL_DIR="$3"
local DL_ROOT_DIR=""
local SAVE_DIR
local CMD=""
SAVE_DIR="$(pwd)"
if [ "$DL_DIR" == "" ]; then
DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')"
fi
echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'"
DL_ROOT_DIR=$(dirname "$DL_DIR")
if [ ! -d "$DL_DIR" ]; then
if [ ! -d "$DL_ROOT_DIR" ]; then
CMD="mkdir -p '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
CMD="cd '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git clone '$GIT_URL' '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
CMD="cd '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git fetch"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git checkout '$BRANCH'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git pull"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
cd "$SAVE_DIR"
return 0
}
#
# dl_bare_git_from_url <git-url> <dir>
#
# Download a bare git from supplied url into desired directory.
#
dl_bare_git_from_url () {
local GIT_URL="$1"
local DL_DIR="$2"
local DL_ROOT_DIR=""
local SAVE_DIR
local CMD=""
SAVE_DIR="$(pwd)"
if [ "$DL_DIR" == "" ]; then
DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')"
fi
echo "dl_bare_git_from_url GIT_URL='$GIT_URL' DL_DIR='$DL_DIR'"
DL_ROOT_DIR=$(dirname "$DL_DIR")
if [ ! -d "$DL_DIR" ]; then
if [ ! -d "$DL_ROOT_DIR" ]; then
CMD="mkdir -p '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
CMD="cd '$DL_ROOT_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git clone --bare '$GIT_URL' '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="cd '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git --bare update-server-info"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
if [ -f hooks/post-update.sample ]; then
CMD="mv -f hooks/post-update.sample hooks/post-update"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
fi
fi
CMD="cd '$DL_DIR'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
CMD="git fetch"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
cd "$SAVE_DIR"
return 1
fi
cd "$SAVE_DIR"
return 0
}
#
# dl_file_from_url <url>
#
# Download a file to the current directory
#
dl_file_from_url () {
local URL="$1"
local DOWNLOAD_PATH=""
local DOWNLOAD_DIR=""
local PROTOCOL=""
local CMD=""
DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")"
DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")"
PROTOCOL=$(url_protocol $URL)
echo "$PROTOCOL $URL $DOWNLOAD_PATH"
if [ -f "$DOWNLOAD_PATH" ]; then
echo "Already have '$DOWNLOAD_PATH'"
return 0
fi
case "$PROTOCOL" in
https|http)
if [ ! -d "$DOWNLOAD_DIR" ]; then
CMD="mkdir -p '$DOWNLOAD_DIR'"
echo "$CMD"
eval "$CMD"
if [ $? -ne 0 ]; then
echo "Error: $CMD"
return 1
fi
fi
CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
return 1
fi
;;
*)
echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'"
return 1
;;
esac
return 0
}

50
centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh → centos-mirror-tools/stx_mirror_scripts/repo_sync.sh
View File

@ -3,7 +3,7 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Daily update script for mirror.starlingx.cengn.ca covering
# Update script for mirror.starlingx.cengn.ca covering
# rpms and src.rpms downloaded from a yum repository.
#
# IMPORTANT: This script is only to be run on the StarlingX mirror.
@ -11,9 +11,11 @@
#
# Configuration files for repositories to be downloaded are currently
# stored at mirror.starlingx.cengn.ca:/export/config/yum.repos.d.
# Those repos were derived from stx-tools/centos-mirror-tools/yum.repos.d
# with some modifications that will need to be automated in a
# future update.
# Those repos were derived from those found in
# stx-tools/centos-mirror-tools/yum.repos.d with some modifications.
#
# It is recommended that repo_update.sh be run prior to this script
# to ensure /export/config/yum.repos.d is kept current.
#
# This script was originated by Scott Little.
#
@ -24,13 +26,6 @@ YUM_REPOS_DIR="$YUM_CONF_DIR/yum.repos.d"
DOWNLOAD_PATH_ROOT="/export/mirror/centos"
URL_UTILS="url_utils.sh"
# These variables drive the download of the centos installer
# and other non-repo files found under the os/x86_64 subdirectory.
OS_PATH_PREFIX=/export/mirror/centos/centos
OS_PATH_SUFFIX=os/x86_64
OS_FILES="EULA GPL"
OS_DIRS="EFI LiveOS images isolinux"
DAILY_REPO_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
if [ -f "$DAILY_REPO_SYNC_DIR/$URL_UTILS" ]; then
@ -62,13 +57,12 @@ if [ ! -f "$YUM_CONF" ]; then
exit 1
fi
for REPO in $(find $YUM_REPOS_DIR -name '*.repo'); do
for REPO_ID in $(grep '^[[]' $REPO | sed 's#[][]##g'); do
for REPO_ID in $(yum repolist --config="$YUM_CONF" --quiet | tail -n +2 | cut -d ' ' -f 1); do
REPO_URL=$(yum repoinfo --config="$YUM_CONF" --disablerepo="*" --enablerepo="$REPO_ID" | grep Repo-baseurl | cut -d ' ' -f 3)
DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$REPO_URL")"
echo "Processing: REPO=$REPO REPO_ID=$REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH"
echo "Processing: REPO_ID=$REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH"
# Assume it's a repo of binary rpms unless repoid ends in
# some variation of 'source'.
@ -125,36 +119,10 @@ for REPO in $(find $YUM_REPOS_DIR -name '*.repo'); do
continue
fi
# The following will download the centos installer and other non-repo
# files and directories found under the os/x86_64 subdirectory.
if [[ "$DOWNLOAD_PATH" == "$OS_PATH_PREFIX"/*/"$OS_PATH_SUFFIX" ]]; then
for f in $OS_FILES; do
CMD="wget '$REPO_URL/$f' --output-document='$DOWNLOAD_PATH/$f'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
ERR_COUNT=$((ERR_COUNT+1))
continue
fi
done
for d in $OS_DIRS; do
CMD="wget -r -N -l 3 -nv -np -e robots=off --reject-regex '.*[?].*' --reject index.html '$REPO_URL/$d/' -P '$OS_PATH_PREFIX/'"
echo "$CMD"
eval $CMD
if [ $? -ne 0 ]; then
echo "Error: $CMD"
ERR_COUNT=$((ERR_COUNT+1))
continue
fi
done
fi
popd
done
done | tee $LOGFILE
echo ERR_COUNT=$ERR_COUNT
if [ $ERR_COUNT -ne 0 ]; then
exit 1
fi

409
centos-mirror-tools/stx_mirror_scripts/repo_update.sh Executable file
View File

@ -0,0 +1,409 @@
#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
# Update script for mirror.starlingx.cengn.ca covering
# yum.repos.d.
#
# IMPORTANT: This script is only to be run on the StarlingX mirror.
# It is not for use by the general StarlinX developer.
#
# Configuration files for repositories to be downloaded are currently
# stored at mirror.starlingx.cengn.ca:/export/config/yum.repos.d/
# and /export/config/rpm-gpg-keys. These configuration files need
# to be updated periodically to reflect changes made to
# stx-tools/centos-mirror-tools/yum.repos.d/ and
# stx-tools/centos-mirror-tools/rpm-gpg-keys/.
# The update are additive in nature, mirror.starlingx.cengn.ca
# does not delete keys or repos. At worst we will rename a
# repo if it's url has changed.
#
# This script was originated by Scott Little.
#
LOGFILE="/export/log/repo_update.log"
YUM_CONF_DIR="/export/config"
# YUM_CONF_DIR="/tmp/config"
YUM_CONF="$YUM_CONF_DIR/yum.conf"
YUM_REPOS_DIR="$YUM_CONF_DIR/yum.repos.d"
GPG_KEYS_DIR="$YUM_CONF_DIR/rpm-gpg-keys"
DOWNLOAD_PATH_ROOT=/export/mirror/centos
STX_TOOLS_BRANCH="master"
STX_TOOLS_BRANCH_ROOT_DIR="$HOME/stx-tools"
STX_TOOLS_OS_SUBDIR="centos-mirror-tools"
DAILY_REPO_DIR_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
if [ -f "$DAILY_REPO_DIR_SYNC_DIR/stx_tool_utils.sh" ]; then
source "$DAILY_REPO_DIR_SYNC_DIR/stx_tool_utils.sh"
elif [ -f "$DAILY_REPO_DIR_SYNC_DIR/../stx_tool_utils.sh" ]; then
source "$DAILY_REPO_DIR_SYNC_DIR/../stx_tool_utils.sh"
else
>&2 echo "Error: Can't find 'stx_tool_utils.sh'"
exit 1
fi
usage () {
echo "$0 [-b <branch>] [-d <dir>]"
echo ""
echo "Options:"
echo " -b: Use an alternate branch of stx-tools. Default is 'master'."
echo " -d: Directory where we will clone stx-tools. Default is \$HOME."
echo ""
}
while getopts "b:d:h" opt; do
case "${opt}" in
b)
# branch
STX_TOOLS_BRANCH="${OPTARG}"
if [ $"STX_TOOLS_BRANCH" == "" ]; then
usage
exit 1
fi
;;
d)
# download directory for stx-tools
STX_TOOLS_BRANCH_ROOT_DIR="${OPTARG}"
if [ "$STX_TOOLS_BRANCH_ROOT_DIR" == "" ]; then
usage
exit 1
fi
;;
h)
# Help
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH"
STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools"
UPSTREAM_YUM_REPOS_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/yum.repos.d"
UPSTREAM_YUM_CONF="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/yum.conf.sample"
UPSTREAM_GPG_KEYS_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/rpm-gpg-keys"
update_gpg_keys () {
local UPSTREAM_KEY=""
local KEY=""
local UPSTREAM_CHECKSUM=""
local CHECKSUM=""
for UPSTREAM_KEY in $(find $UPSTREAM_GPG_KEYS_DIR -type f | sort ); do
KEY=$GPG_KEYS_DIR/$(basename $UPSTREAM_KEY)
if [ ! -f "$KEY" ]; then
echo "Copy new key file '$UPSTREAM_KEY' to '$KEY'"
\cp "$UPSTREAM_KEY" "$KEY"
continue
fi
UPSTREAM_CHECKSUM=$(md5sum $UPSTREAM_KEY | cut -d ' ' -f 1)
CHECKSUM=$(md5sum $KEY | cut -d ' ' -f 1)
if [ "$UPSTREAM_CHECKSUM" == "$CHECKSUM" ]; then
echo "Already have '$UPSTREAM_KEY'"
continue
fi
# Key mismatch. What to do?
>&2 echo "Error: Key mismatch: '$UPSTREAM_KEY' vs '$KEY'"
ERR_COUNT=$((ERR_COUNT + 1))
done
return 0
}
get_repo_url () {
local YUM_CONF="$1"
local REPO_ID="$2"
local URL=""
URL=$(cd $(dirname $YUM_CONF);
yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \
grep Repo-baseurl | \
cut -d ' ' -f 3;
exit ${PIPESTATUS[0]}
)
if [ $? != 0 ]; then
>&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'"
return 1
fi
echo "$URL"
return 0
}
get_repo_name () {
local YUM_CONF="$1"
local REPO_ID="$2"
local NAME=""
NAME=$(cd $(dirname $YUM_CONF);
yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \
grep Repo-name | \
cut -d ' ' -f 3;
exit ${PIPESTATUS[0]}
)
if [ $? != 0 ]; then
>&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'"
return 1
fi
echo "$NAME"
return 0
}
archive_repo_id () {
local REPO_ID="$1"
local YUM_CONF="$2"
local REPO="$3"
local REPO_NAME=""
local TEMP=""
local EXTRA=""
if [ ! -f "$YUM_CONF" ]; then
>&2 echo "ERROR: invalid file YUM_CONF='$YUM_CONF'"
return 1
fi
if [ ! -f "$REPO" ]; then
>&2 echo "ERROR: invalid file REPO='$REPO'"
return 1
fi
REPO_NAME=$(get_repo_name "$YUM_CONF" "$REPO_ID")
if [ $? != 0 ]; then
return 1
fi
TEMP=$(mktemp '/tmp/repo_update_XXXXXX')
if [ "$TEMP" == "" ]; then
>&2 echo "ERROR: mktemp '/tmp/repo_update_XXXXXX'"
return 1
fi
EXTRA=$(echo $TEMP | sed 's#/tmp/repo_update_##')
\rm $TEMP
echo "Archive: '$REPO_ID' as '$REPO_ID-$EXTRA' in file '$REPO'"
sed -i "s#^[[]$REPO_ID[]]#[$REPO_ID-$EXTRA]#" "$REPO"
sed -i "s#^name=$REPO_NAME\$#name=$REPO_NAME-$EXTRA#" "$REPO"
return 0
}
copy_repo_id () {
local REPO_ID="$1"
local FORM_REPO="$2"
local TO_REPO="$3"
local TEMPDIR=""
local FRAGMENT=""
echo "Copy new repo id: '$REPO_ID' from '$FORM_REPO' into file '$TO_REPO'"
if [ ! -f "$FORM_REPO" ]; then
>&2 echo "ERROR: invalid file FORM_REPO='$FORM_REPO'"
return 1
fi
if [ ! -f "$TO_REPO" ]; then
>&2 echo "ERROR: invalid file TO_REPO='$TO_REPO'"
return 1
fi
TEMPDIR=$(mktemp -d '/tmp/repo_update_XXXXXX')
if [ "$TEMPDIR" == "" ]; then
>&2 echo "ERROR: mktemp -d '/tmp/repo_update_XXXXXX'"
return 1
fi
csplit --prefix=$TEMPDIR/xx --quiet "$FORM_REPO" '/^[[]/' '{*}' >> /dev/null
if [ $? -ne 0 ]; then
>&2 echo "ERROR: csplit --prefix=$TEMPDIR/xx '$FORM_REPO' '/^[[]/' '{*}'"
return 1
fi
FRAGMENT=$(grep -l "$REPO_ID" $TEMPDIR/* | head -n 1)
if [ "$TEMPDIR" == "" ]; then
>&2 echo "ERROR: grep -l '$REPO_ID' $TEMPDIR/* | head -n 1"
return 1
fi
echo >> $TO_REPO
cat $FRAGMENT | sed "s#/etc/pki/rpm-gpg#$GPG_KEYS_DIR#" >> $TO_REPO
\rm -rf $TEMPDIR
return 0
}
update_yum_repos_d () {
local UPSTREAM_REPO=""
local REPO=""
local UPSTREAM_REPO_ID=""
local REPO_ID=""
local UPSTREAM_REPO_URL=""
local REPO_URL=""
local UPSTREAM_REPO_NAME=""
local REPO_NAME=""
local UPSTREAM_DOWNLOAD_PATH=""
local DOWNLOAD_PATH=""
local TEMPDIR=""
for UPSTREAM_REPO in $(find $UPSTREAM_YUM_REPOS_DIR -name '*.repo' | sort ); do
REPO=$YUM_REPOS_DIR/$(basename $UPSTREAM_REPO)
if [ ! -f $REPO ]; then
# New repo file
echo "Copy new repo file '$UPSTREAM_REPO' to '$REPO'"
cat "$UPSTREAM_REPO" | sed "s#/etc/pki/rpm-gpg#$GPG_KEYS_DIR#" > "$REPO"
continue
fi
for UPSTREAM_REPO_ID in $(grep '^[[]' $UPSTREAM_REPO | sed 's#[][]##g'); do
UPSTREAM_REPO_URL=$(get_repo_url "$UPSTREAM_YUM_CONF" "$UPSTREAM_REPO_ID")
if [ $? != 0 ]; then
return 1
fi
UPSTREAM_REPO_NAME=$(get_repo_name "$UPSTREAM_YUM_CONF" "$UPSTREAM_REPO_ID")
if [ $? != 0 ]; then
return 1
fi
UPSTREAM_DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$UPSTREAM_REPO_URL")"
# echo "Processing: REPO=$UPSTREAM_REPO REPO_ID=$UPSTREAM_REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH"
REPO_ID=$(grep "^[[]$UPSTREAM_REPO_ID[]]" $REPO | sed 's#[][]##g')
if [ "$REPO_ID" == "" ]; then
copy_repo_id "$UPSTREAM_REPO_ID" "$UPSTREAM_REPO" "$REPO"
if [ $? != 0 ]; then
>&2 echo "Error: copy_repo_id '$UPSTREAM_REPO_ID' '$UPSTREAM_REPO' '$REPO'"
return 1
fi
continue
fi
if [ "$REPO_ID" != "$UPSTREAM_REPO_ID" ]; then
>&2 echo "Error: bad grep? '$REPO_ID' != '$UPSTREAM_REPO_ID'"
return 1
fi
# REPO_URL=$(cd $(dirname $YUM_CONF);
# yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \
# grep Repo-baseurl | \
# cut -d ' ' -f 3;
# exit ${PIPESTATUS[0]})
REPO_URL=$(get_repo_url "$YUM_CONF" "$REPO_ID")
if [ $? != 0 ]; then
# >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'"
return 1
fi
REPO_NAME=$(get_repo_name "$YUM_CONF" "$REPO_ID")
if [ $? != 0 ]; then
# >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'"
return 1
fi
REPO_URL=$(yum repoinfo --config="$YUM_CONF" --disablerepo="*" --enablerepo="$REPO_ID" | grep Repo-baseurl | cut -d ' ' -f 3)
DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$REPO_URL")"
# Check critical content is the same
if [ "$UPSTREAM_REPO_URL" == "$REPO_URL" ] && [ "$UPSTREAM_DOWNLOAD_PATH" == "$DOWNLOAD_PATH" ] && [ "$UPSTREAM_REPO_NAME" == "$REPO_NAME" ]; then
echo "Already have '$UPSTREAM_REPO_ID' from '$UPSTREAM_REPO'"
continue
fi
# Something has changed, log it
if [ "$UPSTREAM_REPO_URL" != "$REPO_URL" ]; then
>&2 echo "Warning: Existing repo has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, url:$REPO_URL -> $UPSTREAM_REPO_URL"
elif [ "$UPSTREAM_REPO_NAME" != "$REPO_NAME" ]; then
>&2 echo "Warning: Existing repo has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, name:$REPO_URL -> $UPSTREAM_REPO_URL"
elif [ "$UPSTREAM_DOWNLOAD_PATH" != "$DOWNLOAD_PATH" ]; then
>&2 echo "Warning: Existing download path has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, path:$UPSTREAM_DOWNLOAD_PATH -> $DOWNLOAD_PATH"
fi
archive_repo_id "$REPO_ID" "$YUM_CONF" "$REPO"
copy_repo_id "$UPSTREAM_REPO_ID" "$UPSTREAM_REPO" "$REPO"
if [ $? != 0 ]; then
>&2 echo "Error: copy_repo_id '$UPSTREAM_REPO_ID' '$UPSTREAM_REPO' '$REPO'"
return 1
fi
# # Create new repo id? Edit old one? Unclear what to do.
# ERR_COUNT=$((ERR_COUNT + 1))
done
done
return 0
}
if [ -f $LOGFILE ]; then
\rm -f $LOGFILE
fi
(
ERR_COUNT=0
mkdir -p "$YUM_CONF_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: mkdir -p '$YUM_CONF_DIR'"
exit 1
fi
mkdir -p "$YUM_REPOS_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: mkdir -p '$YUM_CONF_DIR'"
exit 1
fi
mkdir -p "$GPG_KEYS_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: mkdir -p '$YUM_CONF_DIR'"
exit 1
fi
stx_tool_clone_or_update "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_ROOT_DIR"
if [ $? -ne 0 ]; then
>&2 echo "Error: Failed to update stx_tools. Can't continue."
exit 1
fi
if [ ! -f "$YUM_CONF" ]; then
echo "Copy yum.conf: '$UPSTREAM_YUM_CONF' -> '$YUM_CONF'"
cat $UPSTREAM_YUM_CONF | sed "s#=/tmp/#=$YUM_CONF_DIR/#" | \
sed "s#reposdir=yum.repos.d#reposdir=$YUM_CONF_DIR/yum.repos.d#" | \
sed 's#/etc/pki/rpm-gpg/#$GPG_KEYS_DIR/#' >> $YUM_CONF
fi
update_gpg_keys
if [ $? -ne 0 ]; then
>&2 echo "Error: Failed in update_gpg_keys Can't continue."
exit 1
fi
update_yum_repos_d
if [ $? -ne 0 ]; then
>&2 echo "Error: Failed in update_yum_repos_d. Can't continue."
exit 1
fi
if [ $ERR_COUNT -ne 0 ]; then
>&2 echo "Error: Failed to update $ERR_COUNT repo_id's"
exit 1
fi
exit 0
) | tee $LOGFILE
exit ${PIPESTATUS[0]}

55
centos-mirror-tools/stx_mirror_scripts/stx_tool_utils.sh Normal file
View File

@ -0,0 +1,55 @@
#
# SPDX-License-Identifier: Apache-2.0
#
# Utility functions to download stx-tools git
#
# This script was originated by Scott Little.
#
STX_TOOL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
if [ -f "$STX_TOOL_UTILS_DIR/dl_utils.sh" ]; then
source "$STX_TOOL_UTILS_DIR/dl_utils.sh"
elif [ -f "$STX_TOOL_UTILS_DIR/../dl_utils.sh" ]; then
source "$STX_TOOL_UTILS_DIR/../dl_utils.sh"
else
echo "Error: Can't find 'dl_utils.sh'"
exit 1
fi
STX_TOOLS_DEFAULT_BRANCH="master"
STX_TOOLS_DEFAULT_ROOT_DIR="$HOME/stx-tools"
STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git"
#
# stx_tool_clone_or_update [<branch>] [<dir>]
#
# Clone stx-tools under the supplied directory,
# and checkout the desired branch.
#
stx_tool_clone_or_update () {
local BRANCH="$1"
local DL_ROOT_DIR="$2"
local CMD
if [ "$BRANCH" == "" ]; then
BRANCH="$STX_TOOLS_DEFAULT_BRANCH"
fi
if [ "$DL_ROOT_DIR" == "" ]; then
DL_ROOT_DIR="$STX_TOOLS_DEFAULT_ROOT_DIR/$BRANCH"
fi
local DL_DIR="$DL_ROOT_DIR/stx-tools"
dl_git_from_url "$STX_TOOLS_GIT_URL" "$BRANCH" "$DL_DIR"
if [ $? -ne 0 ]; then
echo "Error: Failed to download '$STX_TOOLS_GIT_URL'"
return 1;
fi
return 0
}

9
centos-mirror-tools/url_utils.sh
View File

@ -1,8 +1,5 @@
#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
#
# A set of bash utility functions to parse a URL.
# This script was originated by Scott Little
@ -16,7 +13,11 @@ url_protocol () {
return 1
fi
echo "$URL" | sed 's#^\(.*\)://.*$#\1#'
if echo "$URL" | grep -q '[:][/][/]' ;then
echo "$URL" | sed 's#^\(.*\)://.*$#\1#'
else
echo "http"
fi
return 0
}