diff --git a/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh b/centos-mirror-tools/stx_mirror_scripts/dl_sync.sh similarity index 56% rename from centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh rename to centos-mirror-tools/stx_mirror_scripts/dl_sync.sh index d100699dd..58f684446 100755 --- a/centos-mirror-tools/stx_mirror_scripts/daily_dl_sync.sh +++ b/centos-mirror-tools/stx_mirror_scripts/dl_sync.sh @@ -3,9 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 # -# Daily update script for mirror.starlingx.cengn.ca covering +# Update script for mirror.starlingx.cengn.ca covering # tarballs and other files not downloaded from a yum repository. -# This script was originated by Scott Little. +# The list of files to download are pulled from the .lst files +# found in the stx-tools repo. # # IMPORTANT: This script is only to be run on the StarlingX mirror. # It is not for use by the general StarlinX developer. @@ -15,24 +16,25 @@ DAILY_DL_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" -if [ -f "$DAILY_DL_SYNC_DIR/url_utils.sh" ]; then - source "$DAILY_DL_SYNC_DIR/url_utils.sh" -elif [ -f "$DAILY_DL_SYNC_DIR/../url_utils.sh" ]; then - source "$DAILY_DL_SYNC_DIR/../url_utils.sh" -else - echo "Error: Can't find 'url_utils.sh'" - exit 1 -fi - - LOGFILE=/export/log/daily_dl_sync.log DOWNLOAD_PATH_ROOT=/export/mirror/centos STX_TOOLS_BRANCH="master" -STX_TOOLS_BRANCH_ROOT_DIR="$HOME" -STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git" +STX_TOOLS_BRANCH_ROOT_DIR="$HOME/stx-tools" STX_TOOLS_OS_SUBDIR="centos-mirror-tools" +if [ -f "$DAILY_DL_SYNC_DIR/stx_tool_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/stx_tool_utils.sh" +elif [ -f "$DAILY_DL_SYNC_DIR/../stx_tool_utils.sh" ]; then + source "$DAILY_DL_SYNC_DIR/../stx_tool_utils.sh" +else + echo "Error: Can't find 'stx_tool_utils.sh'" + exit 1 +fi + + + + usage () { echo "$0 [-b ] [-d ]" echo "" @@ -76,151 +78,6 @@ STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH" STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools" LST_FILE_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR" -dl_git_from_url () { - local GIT_URL="$1" - local BRANCH="$2" - local DL_DIR="$3" - local DL_ROOT_DIR="" - local SAVE_DIR - local CMD="" - - SAVE_DIR="$(pwd)" - - if [ "$DL_DIR" == "" ]; then - DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')" - fi - - echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'" - DL_ROOT_DIR=$(dirname "$DL_DIR") - - if [ ! -d "$DL_DIR" ]; then - if [ ! -d "$DL_ROOT_DIR" ]; then - CMD="mkdir -p '$DL_ROOT_DIR'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - fi - - CMD="cd '$DL_ROOT_DIR'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - CMD="git clone --bare '$GIT_URL' '$DL_DIR'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - CMD="cd '$DL_DIR'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - CMD="git --bare update-server-info" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - if [ -f hooks/post-update.sample ]; then - CMD="mv -f hooks/post-update.sample hooks/post-update" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - fi - fi - - CMD="cd '$DL_DIR'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - CMD="git fetch" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - cd "$SAVE_DIR" - return 1 - fi - - cd "$SAVE_DIR" - return 0 -} - - -dl_file_from_url () { - local URL="$1" - local DOWNLOAD_PATH="" - local DOWNLOAD_DIR="" - local PROTOCOL="" - local CMD="" - - DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")" - DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")" - PROTOCOL=$(url_protocol $URL) - echo "$PROTOCOL $URL $DOWNLOAD_PATH" - - if [ -f "$DOWNLOAD_PATH" ]; then - echo "Already have '$DOWNLOAD_PATH'" - return 0 - fi - - case "$PROTOCOL" in - https|http) - if [ ! -d "$DOWNLOAD_DIR" ]; then - CMD="mkdir -p '$DOWNLOAD_DIR'" - echo "$CMD" - eval "$CMD" - if [ $? -ne 0 ]; then - echo "Error: $CMD" - return 1 - fi - fi - - CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - return 1 - fi - ;; - *) - echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'" - ;; - esac - - return 0 -} - raw_dl_from_rpm_lst () { local FILE="$1" @@ -238,6 +95,7 @@ raw_dl_from_rpm_lst () { return $ERR_COUNT } + raw_dl_from_non_rpm_lst () { local FILE="$1" local TAR="" @@ -290,7 +148,7 @@ raw_dl_from_non_rpm_lst () { ;; git|git_script) BRANCH="$UTIL" - dl_git_from_url "$URL" "$BRANCH" "" + dl_bare_git_from_url "$URL" "" if [ $? -ne 0 ]; then echo "Error: Failed to download '$URL' while processing '$TARBALL_NAME'" ERR_COUNT=$((ERR_COUNT+1)) @@ -314,24 +172,6 @@ raw_dl_from_non_rpm_lst () { } -stx_tool_clone_or_update () { - local CMD - - CMD="mkdir -p '$STX_TOOLS_DL_DIR'" - echo "$CMD" - eval "$CMD" - if [ $? -ne 0 ]; then - echo "Error: $CMD" - return 1 - fi - - dl_git_from_url "$STX_TOOLS_GIT_URL" "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to download '$STX_TOOLS_GIT_URL'" - return 1; - fi - return 0 -} if [ -f $LOGFILE ]; then @@ -341,7 +181,7 @@ fi ( ERR_COUNT=0 -stx_tool_clone_or_update +stx_tool_clone_or_update "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_ROOT_DIR" if [ $? -ne 0 ]; then echo "Error: Failed to update stx_tools. Can't continue." exit 1 diff --git a/centos-mirror-tools/stx_mirror_scripts/dl_utils.sh b/centos-mirror-tools/stx_mirror_scripts/dl_utils.sh new file mode 100644 index 000000000..afbbe75bc --- /dev/null +++ b/centos-mirror-tools/stx_mirror_scripts/dl_utils.sh @@ -0,0 +1,277 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Utility function for the download of gits and tarballs. +# +# This script was originated by Scott Little. +# + +DL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +if [ -f "$DL_UTILS_DIR/url_utils.sh" ]; then + source "$DL_UTILS_DIR/url_utils.sh" +elif [ -f "$DL_UTILS_DIR/../url_utils.sh" ]; then + source "$DL_UTILS_DIR/../url_utils.sh" +else + echo "Error: Can't find 'url_utils.sh'" + exit 1 +fi + + +DOWNLOAD_PATH_ROOT=${DOWNLOAD_PATH_ROOT:-/export/mirror/centos} + + +# +# dl_git_from_url +# +# Download a git from supplied url into directory, +# and checkout desired branch. +# +dl_git_from_url () { + local GIT_URL="$1" + local BRANCH="$2" + local DL_DIR="$3" + + local DL_ROOT_DIR="" + local SAVE_DIR + local CMD="" + + SAVE_DIR="$(pwd)" + + if [ "$DL_DIR" == "" ]; then + DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')" + fi + + echo "dl_git_from_url GIT_URL='$GIT_URL' BRANCH='$BRANCH' DL_DIR='$DL_DIR'" + DL_ROOT_DIR=$(dirname "$DL_DIR") + + if [ ! -d "$DL_DIR" ]; then + if [ ! -d "$DL_ROOT_DIR" ]; then + CMD="mkdir -p '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + + CMD="cd '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git clone '$GIT_URL' '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git fetch" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git checkout '$BRANCH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git pull" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + cd "$SAVE_DIR" + return 0 +} + + +# +# dl_bare_git_from_url +# +# Download a bare git from supplied url into desired directory. +# +dl_bare_git_from_url () { + local GIT_URL="$1" + local DL_DIR="$2" + + local DL_ROOT_DIR="" + local SAVE_DIR + local CMD="" + + SAVE_DIR="$(pwd)" + + if [ "$DL_DIR" == "" ]; then + DL_DIR="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$GIT_URL" | sed 's#[.]git$##')" + fi + + echo "dl_bare_git_from_url GIT_URL='$GIT_URL' DL_DIR='$DL_DIR'" + DL_ROOT_DIR=$(dirname "$DL_DIR") + + if [ ! -d "$DL_DIR" ]; then + if [ ! -d "$DL_ROOT_DIR" ]; then + CMD="mkdir -p '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + + CMD="cd '$DL_ROOT_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git clone --bare '$GIT_URL' '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git --bare update-server-info" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + if [ -f hooks/post-update.sample ]; then + CMD="mv -f hooks/post-update.sample hooks/post-update" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + fi + fi + + CMD="cd '$DL_DIR'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + CMD="git fetch" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + cd "$SAVE_DIR" + return 1 + fi + + cd "$SAVE_DIR" + return 0 +} + + +# +# dl_file_from_url +# +# Download a file to the current directory +# +dl_file_from_url () { + local URL="$1" + + local DOWNLOAD_PATH="" + local DOWNLOAD_DIR="" + local PROTOCOL="" + local CMD="" + + DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$URL")" + DOWNLOAD_DIR="$(dirname "$DOWNLOAD_PATH")" + PROTOCOL=$(url_protocol $URL) + echo "$PROTOCOL $URL $DOWNLOAD_PATH" + + if [ -f "$DOWNLOAD_PATH" ]; then + echo "Already have '$DOWNLOAD_PATH'" + return 0 + fi + + case "$PROTOCOL" in + https|http) + if [ ! -d "$DOWNLOAD_DIR" ]; then + CMD="mkdir -p '$DOWNLOAD_DIR'" + echo "$CMD" + eval "$CMD" + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + fi + + CMD="wget '$URL' --tries=5 --wait=15 --output-document='$DOWNLOAD_PATH'" + echo "$CMD" + eval $CMD + if [ $? -ne 0 ]; then + echo "Error: $CMD" + return 1 + fi + ;; + *) + echo "Error: Unknown protocol '$PROTOCOL' for url '$URL'" + return 1 + ;; + esac + + return 0 +} + + diff --git a/centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh b/centos-mirror-tools/stx_mirror_scripts/repo_sync.sh similarity index 65% rename from centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh rename to centos-mirror-tools/stx_mirror_scripts/repo_sync.sh index 666f56e4c..66bb1dc24 100755 --- a/centos-mirror-tools/stx_mirror_scripts/daily_repo_sync.sh +++ b/centos-mirror-tools/stx_mirror_scripts/repo_sync.sh @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 # -# Daily update script for mirror.starlingx.cengn.ca covering +# Update script for mirror.starlingx.cengn.ca covering # rpms and src.rpms downloaded from a yum repository. # # IMPORTANT: This script is only to be run on the StarlingX mirror. @@ -11,9 +11,11 @@ # # Configuration files for repositories to be downloaded are currently # stored at mirror.starlingx.cengn.ca:/export/config/yum.repos.d. -# Those repos were derived from stx-tools/centos-mirror-tools/yum.repos.d -# with some modifications that will need to be automated in a -# future update. +# Those repos were derived from those found in +# stx-tools/centos-mirror-tools/yum.repos.d with some modifications. +# +# It is recommended that repo_update.sh be run prior to this script +# to ensure /export/config/yum.repos.d is kept current. # # This script was originated by Scott Little. # @@ -24,13 +26,6 @@ YUM_REPOS_DIR="$YUM_CONF_DIR/yum.repos.d" DOWNLOAD_PATH_ROOT="/export/mirror/centos" URL_UTILS="url_utils.sh" -# These variables drive the download of the centos installer -# and other non-repo files found under the os/x86_64 subdirectory. -OS_PATH_PREFIX=/export/mirror/centos/centos -OS_PATH_SUFFIX=os/x86_64 -OS_FILES="EULA GPL" -OS_DIRS="EFI LiveOS images isolinux" - DAILY_REPO_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" if [ -f "$DAILY_REPO_SYNC_DIR/$URL_UTILS" ]; then @@ -62,13 +57,12 @@ if [ ! -f "$YUM_CONF" ]; then exit 1 fi -for REPO in $(find $YUM_REPOS_DIR -name '*.repo'); do - for REPO_ID in $(grep '^[[]' $REPO | sed 's#[][]##g'); do +for REPO_ID in $(yum repolist --config="$YUM_CONF" --quiet | tail -n +2 | cut -d ' ' -f 1); do REPO_URL=$(yum repoinfo --config="$YUM_CONF" --disablerepo="*" --enablerepo="$REPO_ID" | grep Repo-baseurl | cut -d ' ' -f 3) DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$REPO_URL")" - echo "Processing: REPO=$REPO REPO_ID=$REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH" + echo "Processing: REPO_ID=$REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH" # Assume it's a repo of binary rpms unless repoid ends in # some variation of 'source'. @@ -125,36 +119,10 @@ for REPO in $(find $YUM_REPOS_DIR -name '*.repo'); do continue fi - # The following will download the centos installer and other non-repo - # files and directories found under the os/x86_64 subdirectory. - if [[ "$DOWNLOAD_PATH" == "$OS_PATH_PREFIX"/*/"$OS_PATH_SUFFIX" ]]; then - for f in $OS_FILES; do - CMD="wget '$REPO_URL/$f' --output-document='$DOWNLOAD_PATH/$f'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - ERR_COUNT=$((ERR_COUNT+1)) - continue - fi - done - - for d in $OS_DIRS; do - CMD="wget -r -N -l 3 -nv -np -e robots=off --reject-regex '.*[?].*' --reject index.html '$REPO_URL/$d/' -P '$OS_PATH_PREFIX/'" - echo "$CMD" - eval $CMD - if [ $? -ne 0 ]; then - echo "Error: $CMD" - ERR_COUNT=$((ERR_COUNT+1)) - continue - fi - done - fi - popd - done done | tee $LOGFILE +echo ERR_COUNT=$ERR_COUNT if [ $ERR_COUNT -ne 0 ]; then exit 1 fi diff --git a/centos-mirror-tools/stx_mirror_scripts/repo_update.sh b/centos-mirror-tools/stx_mirror_scripts/repo_update.sh new file mode 100755 index 000000000..6b753c022 --- /dev/null +++ b/centos-mirror-tools/stx_mirror_scripts/repo_update.sh @@ -0,0 +1,409 @@ +#!/bin/bash + +# +# SPDX-License-Identifier: Apache-2.0 +# +# Update script for mirror.starlingx.cengn.ca covering +# yum.repos.d. +# +# IMPORTANT: This script is only to be run on the StarlingX mirror. +# It is not for use by the general StarlinX developer. +# +# Configuration files for repositories to be downloaded are currently +# stored at mirror.starlingx.cengn.ca:/export/config/yum.repos.d/ +# and /export/config/rpm-gpg-keys. These configuration files need +# to be updated periodically to reflect changes made to +# stx-tools/centos-mirror-tools/yum.repos.d/ and +# stx-tools/centos-mirror-tools/rpm-gpg-keys/. +# The update are additive in nature, mirror.starlingx.cengn.ca +# does not delete keys or repos. At worst we will rename a +# repo if it's url has changed. +# +# This script was originated by Scott Little. +# + +LOGFILE="/export/log/repo_update.log" +YUM_CONF_DIR="/export/config" +# YUM_CONF_DIR="/tmp/config" +YUM_CONF="$YUM_CONF_DIR/yum.conf" +YUM_REPOS_DIR="$YUM_CONF_DIR/yum.repos.d" +GPG_KEYS_DIR="$YUM_CONF_DIR/rpm-gpg-keys" +DOWNLOAD_PATH_ROOT=/export/mirror/centos +STX_TOOLS_BRANCH="master" +STX_TOOLS_BRANCH_ROOT_DIR="$HOME/stx-tools" +STX_TOOLS_OS_SUBDIR="centos-mirror-tools" + + +DAILY_REPO_DIR_SYNC_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +if [ -f "$DAILY_REPO_DIR_SYNC_DIR/stx_tool_utils.sh" ]; then + source "$DAILY_REPO_DIR_SYNC_DIR/stx_tool_utils.sh" +elif [ -f "$DAILY_REPO_DIR_SYNC_DIR/../stx_tool_utils.sh" ]; then + source "$DAILY_REPO_DIR_SYNC_DIR/../stx_tool_utils.sh" +else + >&2 echo "Error: Can't find 'stx_tool_utils.sh'" + exit 1 +fi + + + + +usage () { + echo "$0 [-b ] [-d ]" + echo "" + echo "Options:" + echo " -b: Use an alternate branch of stx-tools. Default is 'master'." + echo " -d: Directory where we will clone stx-tools. Default is \$HOME." + echo "" +} + +while getopts "b:d:h" opt; do + case "${opt}" in + b) + # branch + STX_TOOLS_BRANCH="${OPTARG}" + if [ $"STX_TOOLS_BRANCH" == "" ]; then + usage + exit 1 + fi + ;; + d) + # download directory for stx-tools + STX_TOOLS_BRANCH_ROOT_DIR="${OPTARG}" + if [ "$STX_TOOLS_BRANCH_ROOT_DIR" == "" ]; then + usage + exit 1 + fi + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + + +STX_TOOLS_DL_ROOT_DIR="$STX_TOOLS_BRANCH_ROOT_DIR/$STX_TOOLS_BRANCH" +STX_TOOLS_DL_DIR="$STX_TOOLS_DL_ROOT_DIR/stx-tools" +UPSTREAM_YUM_REPOS_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/yum.repos.d" +UPSTREAM_YUM_CONF="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/yum.conf.sample" +UPSTREAM_GPG_KEYS_DIR="$STX_TOOLS_DL_DIR/$STX_TOOLS_OS_SUBDIR/rpm-gpg-keys" + + +update_gpg_keys () { + local UPSTREAM_KEY="" + local KEY="" + local UPSTREAM_CHECKSUM="" + local CHECKSUM="" + + for UPSTREAM_KEY in $(find $UPSTREAM_GPG_KEYS_DIR -type f | sort ); do + KEY=$GPG_KEYS_DIR/$(basename $UPSTREAM_KEY) + if [ ! -f "$KEY" ]; then + echo "Copy new key file '$UPSTREAM_KEY' to '$KEY'" + \cp "$UPSTREAM_KEY" "$KEY" + continue + fi + + UPSTREAM_CHECKSUM=$(md5sum $UPSTREAM_KEY | cut -d ' ' -f 1) + CHECKSUM=$(md5sum $KEY | cut -d ' ' -f 1) + if [ "$UPSTREAM_CHECKSUM" == "$CHECKSUM" ]; then + echo "Already have '$UPSTREAM_KEY'" + continue + fi + + # Key mismatch. What to do? + >&2 echo "Error: Key mismatch: '$UPSTREAM_KEY' vs '$KEY'" + ERR_COUNT=$((ERR_COUNT + 1)) + done + + return 0 +} + +get_repo_url () { + local YUM_CONF="$1" + local REPO_ID="$2" + local URL="" + + URL=$(cd $(dirname $YUM_CONF); + yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \ + grep Repo-baseurl | \ + cut -d ' ' -f 3; + exit ${PIPESTATUS[0]} + ) + if [ $? != 0 ]; then + >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'" + return 1 + fi + + echo "$URL" + return 0 +} + +get_repo_name () { + local YUM_CONF="$1" + local REPO_ID="$2" + local NAME="" + + NAME=$(cd $(dirname $YUM_CONF); + yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \ + grep Repo-name | \ + cut -d ' ' -f 3; + exit ${PIPESTATUS[0]} + ) + if [ $? != 0 ]; then + >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'" + return 1 + fi + + echo "$NAME" + return 0 +} + +archive_repo_id () { + local REPO_ID="$1" + local YUM_CONF="$2" + local REPO="$3" + local REPO_NAME="" + local TEMP="" + local EXTRA="" + + if [ ! -f "$YUM_CONF" ]; then + >&2 echo "ERROR: invalid file YUM_CONF='$YUM_CONF'" + return 1 + fi + + if [ ! -f "$REPO" ]; then + >&2 echo "ERROR: invalid file REPO='$REPO'" + return 1 + fi + + REPO_NAME=$(get_repo_name "$YUM_CONF" "$REPO_ID") + if [ $? != 0 ]; then + return 1 + fi + + TEMP=$(mktemp '/tmp/repo_update_XXXXXX') + if [ "$TEMP" == "" ]; then + >&2 echo "ERROR: mktemp '/tmp/repo_update_XXXXXX'" + return 1 + fi + EXTRA=$(echo $TEMP | sed 's#/tmp/repo_update_##') + \rm $TEMP + + echo "Archive: '$REPO_ID' as '$REPO_ID-$EXTRA' in file '$REPO'" + + sed -i "s#^[[]$REPO_ID[]]#[$REPO_ID-$EXTRA]#" "$REPO" + sed -i "s#^name=$REPO_NAME\$#name=$REPO_NAME-$EXTRA#" "$REPO" + return 0 +} + +copy_repo_id () { + local REPO_ID="$1" + local FORM_REPO="$2" + local TO_REPO="$3" + local TEMPDIR="" + local FRAGMENT="" + + echo "Copy new repo id: '$REPO_ID' from '$FORM_REPO' into file '$TO_REPO'" + + if [ ! -f "$FORM_REPO" ]; then + >&2 echo "ERROR: invalid file FORM_REPO='$FORM_REPO'" + return 1 + fi + + if [ ! -f "$TO_REPO" ]; then + >&2 echo "ERROR: invalid file TO_REPO='$TO_REPO'" + return 1 + fi + + TEMPDIR=$(mktemp -d '/tmp/repo_update_XXXXXX') + if [ "$TEMPDIR" == "" ]; then + >&2 echo "ERROR: mktemp -d '/tmp/repo_update_XXXXXX'" + return 1 + fi + + csplit --prefix=$TEMPDIR/xx --quiet "$FORM_REPO" '/^[[]/' '{*}' >> /dev/null + if [ $? -ne 0 ]; then + >&2 echo "ERROR: csplit --prefix=$TEMPDIR/xx '$FORM_REPO' '/^[[]/' '{*}'" + return 1 + fi + + FRAGMENT=$(grep -l "$REPO_ID" $TEMPDIR/* | head -n 1) + if [ "$TEMPDIR" == "" ]; then + >&2 echo "ERROR: grep -l '$REPO_ID' $TEMPDIR/* | head -n 1" + return 1 + fi + + echo >> $TO_REPO + cat $FRAGMENT | sed "s#/etc/pki/rpm-gpg#$GPG_KEYS_DIR#" >> $TO_REPO + \rm -rf $TEMPDIR + return 0 +} + +update_yum_repos_d () { + local UPSTREAM_REPO="" + local REPO="" + local UPSTREAM_REPO_ID="" + local REPO_ID="" + local UPSTREAM_REPO_URL="" + local REPO_URL="" + local UPSTREAM_REPO_NAME="" + local REPO_NAME="" + local UPSTREAM_DOWNLOAD_PATH="" + local DOWNLOAD_PATH="" + local TEMPDIR="" + + for UPSTREAM_REPO in $(find $UPSTREAM_YUM_REPOS_DIR -name '*.repo' | sort ); do + REPO=$YUM_REPOS_DIR/$(basename $UPSTREAM_REPO) + if [ ! -f $REPO ]; then + # New repo file + echo "Copy new repo file '$UPSTREAM_REPO' to '$REPO'" + cat "$UPSTREAM_REPO" | sed "s#/etc/pki/rpm-gpg#$GPG_KEYS_DIR#" > "$REPO" + continue + fi + + for UPSTREAM_REPO_ID in $(grep '^[[]' $UPSTREAM_REPO | sed 's#[][]##g'); do + UPSTREAM_REPO_URL=$(get_repo_url "$UPSTREAM_YUM_CONF" "$UPSTREAM_REPO_ID") + if [ $? != 0 ]; then + return 1 + fi + + UPSTREAM_REPO_NAME=$(get_repo_name "$UPSTREAM_YUM_CONF" "$UPSTREAM_REPO_ID") + if [ $? != 0 ]; then + return 1 + fi + + UPSTREAM_DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$UPSTREAM_REPO_URL")" + + # echo "Processing: REPO=$UPSTREAM_REPO REPO_ID=$UPSTREAM_REPO_ID REPO_URL=$REPO_URL DOWNLOAD_PATH=$DOWNLOAD_PATH" + + REPO_ID=$(grep "^[[]$UPSTREAM_REPO_ID[]]" $REPO | sed 's#[][]##g') + + if [ "$REPO_ID" == "" ]; then + copy_repo_id "$UPSTREAM_REPO_ID" "$UPSTREAM_REPO" "$REPO" + if [ $? != 0 ]; then + >&2 echo "Error: copy_repo_id '$UPSTREAM_REPO_ID' '$UPSTREAM_REPO' '$REPO'" + return 1 + fi + continue + fi + + if [ "$REPO_ID" != "$UPSTREAM_REPO_ID" ]; then + >&2 echo "Error: bad grep? '$REPO_ID' != '$UPSTREAM_REPO_ID'" + return 1 + fi + + # REPO_URL=$(cd $(dirname $YUM_CONF); + # yum repoinfo --config="$(basename $YUM_CONF)" --disablerepo="*" --enablerepo="$REPO_ID" | \ + # grep Repo-baseurl | \ + # cut -d ' ' -f 3; + # exit ${PIPESTATUS[0]}) + REPO_URL=$(get_repo_url "$YUM_CONF" "$REPO_ID") + if [ $? != 0 ]; then + # >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'" + return 1 + fi + + REPO_NAME=$(get_repo_name "$YUM_CONF" "$REPO_ID") + if [ $? != 0 ]; then + # >&2 echo "ERROR: yum repoinfo --config='$YUM_CONF' --disablerepo='*' --enablerepo='$REPO_ID'" + return 1 + fi + + REPO_URL=$(yum repoinfo --config="$YUM_CONF" --disablerepo="*" --enablerepo="$REPO_ID" | grep Repo-baseurl | cut -d ' ' -f 3) + DOWNLOAD_PATH="$DOWNLOAD_PATH_ROOT/$(repo_url_to_sub_path "$REPO_URL")" + + # Check critical content is the same + if [ "$UPSTREAM_REPO_URL" == "$REPO_URL" ] && [ "$UPSTREAM_DOWNLOAD_PATH" == "$DOWNLOAD_PATH" ] && [ "$UPSTREAM_REPO_NAME" == "$REPO_NAME" ]; then + echo "Already have '$UPSTREAM_REPO_ID' from '$UPSTREAM_REPO'" + continue + fi + + # Something has changed, log it + if [ "$UPSTREAM_REPO_URL" != "$REPO_URL" ]; then + >&2 echo "Warning: Existing repo has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, url:$REPO_URL -> $UPSTREAM_REPO_URL" + elif [ "$UPSTREAM_REPO_NAME" != "$REPO_NAME" ]; then + >&2 echo "Warning: Existing repo has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, name:$REPO_URL -> $UPSTREAM_REPO_URL" + elif [ "$UPSTREAM_DOWNLOAD_PATH" != "$DOWNLOAD_PATH" ]; then + >&2 echo "Warning: Existing download path has changed: file:$UPSTREAM_REPO, id:$UPSTREAM_REPO_ID, path:$UPSTREAM_DOWNLOAD_PATH -> $DOWNLOAD_PATH" + fi + + archive_repo_id "$REPO_ID" "$YUM_CONF" "$REPO" + copy_repo_id "$UPSTREAM_REPO_ID" "$UPSTREAM_REPO" "$REPO" + if [ $? != 0 ]; then + >&2 echo "Error: copy_repo_id '$UPSTREAM_REPO_ID' '$UPSTREAM_REPO' '$REPO'" + return 1 + fi + # # Create new repo id? Edit old one? Unclear what to do. + # ERR_COUNT=$((ERR_COUNT + 1)) + done + done + + return 0 +} + + +if [ -f $LOGFILE ]; then + \rm -f $LOGFILE +fi + +( +ERR_COUNT=0 + +mkdir -p "$YUM_CONF_DIR" +if [ $? -ne 0 ]; then + >&2 echo "Error: mkdir -p '$YUM_CONF_DIR'" + exit 1 +fi + +mkdir -p "$YUM_REPOS_DIR" +if [ $? -ne 0 ]; then + >&2 echo "Error: mkdir -p '$YUM_CONF_DIR'" + exit 1 +fi + +mkdir -p "$GPG_KEYS_DIR" +if [ $? -ne 0 ]; then + >&2 echo "Error: mkdir -p '$YUM_CONF_DIR'" + exit 1 +fi + +stx_tool_clone_or_update "$STX_TOOLS_BRANCH" "$STX_TOOLS_DL_ROOT_DIR" +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed to update stx_tools. Can't continue." + exit 1 +fi + +if [ ! -f "$YUM_CONF" ]; then + echo "Copy yum.conf: '$UPSTREAM_YUM_CONF' -> '$YUM_CONF'" + cat $UPSTREAM_YUM_CONF | sed "s#=/tmp/#=$YUM_CONF_DIR/#" | \ + sed "s#reposdir=yum.repos.d#reposdir=$YUM_CONF_DIR/yum.repos.d#" | \ + sed 's#/etc/pki/rpm-gpg/#$GPG_KEYS_DIR/#' >> $YUM_CONF +fi + +update_gpg_keys +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed in update_gpg_keys Can't continue." + exit 1 +fi + +update_yum_repos_d +if [ $? -ne 0 ]; then + >&2 echo "Error: Failed in update_yum_repos_d. Can't continue." + exit 1 +fi + +if [ $ERR_COUNT -ne 0 ]; then + >&2 echo "Error: Failed to update $ERR_COUNT repo_id's" + exit 1 +fi + +exit 0 +) | tee $LOGFILE + +exit ${PIPESTATUS[0]} diff --git a/centos-mirror-tools/stx_mirror_scripts/stx_tool_utils.sh b/centos-mirror-tools/stx_mirror_scripts/stx_tool_utils.sh new file mode 100644 index 000000000..b3426919c --- /dev/null +++ b/centos-mirror-tools/stx_mirror_scripts/stx_tool_utils.sh @@ -0,0 +1,55 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# +# Utility functions to download stx-tools git +# +# This script was originated by Scott Little. +# + +STX_TOOL_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +if [ -f "$STX_TOOL_UTILS_DIR/dl_utils.sh" ]; then + source "$STX_TOOL_UTILS_DIR/dl_utils.sh" +elif [ -f "$STX_TOOL_UTILS_DIR/../dl_utils.sh" ]; then + source "$STX_TOOL_UTILS_DIR/../dl_utils.sh" +else + echo "Error: Can't find 'dl_utils.sh'" + exit 1 +fi + + +STX_TOOLS_DEFAULT_BRANCH="master" +STX_TOOLS_DEFAULT_ROOT_DIR="$HOME/stx-tools" +STX_TOOLS_GIT_URL="https://git.starlingx.io/stx-tools.git" + +# +# stx_tool_clone_or_update [] [] +# +# Clone stx-tools under the supplied directory, +# and checkout the desired branch. +# + +stx_tool_clone_or_update () { + local BRANCH="$1" + local DL_ROOT_DIR="$2" + local CMD + + if [ "$BRANCH" == "" ]; then + BRANCH="$STX_TOOLS_DEFAULT_BRANCH" + fi + + if [ "$DL_ROOT_DIR" == "" ]; then + DL_ROOT_DIR="$STX_TOOLS_DEFAULT_ROOT_DIR/$BRANCH" + fi + + local DL_DIR="$DL_ROOT_DIR/stx-tools" + + dl_git_from_url "$STX_TOOLS_GIT_URL" "$BRANCH" "$DL_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to download '$STX_TOOLS_GIT_URL'" + return 1; + fi + + return 0 +} + diff --git a/centos-mirror-tools/url_utils.sh b/centos-mirror-tools/url_utils.sh index a6df22d37..3122d033a 100755 --- a/centos-mirror-tools/url_utils.sh +++ b/centos-mirror-tools/url_utils.sh @@ -1,8 +1,5 @@ -#!/bin/bash # # SPDX-License-Identifier: Apache-2.0 -# - # # A set of bash utility functions to parse a URL. # This script was originated by Scott Little @@ -16,7 +13,11 @@ url_protocol () { return 1 fi - echo "$URL" | sed 's#^\(.*\)://.*$#\1#' + if echo "$URL" | grep -q '[:][/][/]' ;then + echo "$URL" | sed 's#^\(.*\)://.*$#\1#' + else + echo "http" + fi return 0 }