From 06537894b2bd9be41dc6d9537d41e0a81b47db7e Mon Sep 17 00:00:00 2001 From: Scott Little Date: Fri, 10 Dec 2021 16:06:00 -0500 Subject: [PATCH] branching: Add retries to git push/review commands Review.opendev.org seem unreliable when hit with a lot of transactions in a short time. This means that an attempt to create a release branch, e.g. r/stx.6.0 failes repeatedly, and requires a lot of manual repair. By adding retires and delays, we greatly improve our odds of success. Closes-bug: 1954536 Signed-off-by: Scott Little Change-Id: Ia03ed58b67c02e6984a7e0ee01c6666027df6cbe --- .../branching/create_branches_and_tags.sh | 17 +-- build-tools/branching/create_tags.sh | 6 +- build-tools/branching/push_branches_tags.sh | 52 ++++----- build-tools/branching/push_tags.sh | 14 ++- build-tools/git-repo-utils.sh | 8 +- build-tools/git-utils.sh | 14 +-- build-tools/repo-utils.sh | 12 +- build-tools/utils.sh | 107 ++++++++++++++++++ 8 files changed, 166 insertions(+), 64 deletions(-) create mode 100644 build-tools/utils.sh diff --git a/build-tools/branching/create_branches_and_tags.sh b/build-tools/branching/create_branches_and_tags.sh index 21622964..9e694935 100755 --- a/build-tools/branching/create_branches_and_tags.sh +++ b/build-tools/branching/create_branches_and_tags.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -215,18 +215,21 @@ update_gitreview () { fi if [ $need_commit -eq 1 ]; then + echo_stderr "determinging git review method in ${DIR}" review_method=$(git_repo_review_method) if [ "${review_method}" == "gerrit" ] ; then - timeout 15 git review -s + echo_stderr "running git review -s in ${DIR}" + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 if [ $? != 0 ] ; then if [ ${new_host} -eq 0 ]; then - echo_stderr "ERROR: failed to setup git review in ${DIR}" + echo_stderr "ERROR: 3: failed to setup git review in ${DIR}" exit 1 fi need_rm=1 message="Delete .gitreview for ${branch}" fi + echo_stderr "finished git review -s in ${DIR}" else need_rm=1 message="Delete .gitreview for ${branch}" @@ -338,9 +341,9 @@ for subgit in $SUBGITS; do review_method=$(git_repo_review_method) if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then - git review -s > /dev/null + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 if [ $? != 0 ] ; then - echo_stderr "ERROR: failed to setup git review in ${subgit}" + echo_stderr "ERROR: 1: failed to setup git review in ${subgit}" exit 1 fi fi @@ -409,9 +412,9 @@ if [ $MANIFEST -eq 1 ]; then review_method=$(git_review_method) if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then - git review -s > /dev/null + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 if [ $? != 0 ] ; then - echo_stderr "ERROR: failed to setup git review in ${new_manifest_dir}" + echo_stderr "ERROR: 2: failed to setup git review in ${new_manifest_dir}" exit 1 fi fi diff --git a/build-tools/branching/create_tags.sh b/build-tools/branching/create_tags.sh index 37cf1721..57897d4e 100755 --- a/build-tools/branching/create_tags.sh +++ b/build-tools/branching/create_tags.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -190,7 +190,7 @@ for subgit in $SUBGITS; do review_method=$(git_repo_review_method) if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then - git review -s > /dev/null + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 if [ $? != 0 ] ; then echo_stderr "ERROR: failed to setup git review in ${subgit}" exit 1 @@ -221,7 +221,7 @@ if [ $MANIFEST -eq 1 ]; then review_method=$(git_review_method) if [ -f .gitreview ] && [ "${review_method}" == "gerrit" ] ; then - git review -s > /dev/null + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 if [ $? != 0 ] ; then echo_stderr "ERROR: failed to setup git review in ${new_manifest_dir}" exit 1 diff --git a/build-tools/branching/push_branches_tags.sh b/build-tools/branching/push_branches_tags.sh index 23f2feb3..49e5ee06 100755 --- a/build-tools/branching/push_branches_tags.sh +++ b/build-tools/branching/push_branches_tags.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -224,26 +224,25 @@ for subgit in $SUBGITS; do port=$(url_port "${url}") path=$(url_path "${url}") if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then - echo "git push ${review_remote} ${tag} && \\" - echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\" - echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\" - echo "git review --topic=${branch}" - - git push $DRY_RUN ${review_remote} ${tag} && \ + echo "git push ${review_remote} ${tag}" && \ + with_retries -d 45 -t 15 -k 5 5 git push $DRY_RUN ${review_remote} ${tag} && \ + echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag}" && \ $DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch}" && \ $DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ - $DRY_RUN_CMD git review --topic="${branch}" + echo "git review --topic=${branch/\//.}" && \ + $DRY_RUN_CMD with_retries -d 45 -t 15 -k 5 5 git review --topic="${branch/\//.}" else - echo "git push ${review_remote} ${branch}:${branch} $DRY_RUN" - git push ${review_remote} ${branch}:${branch} $DRY_RUN - echo "git push ${review_remote} ${tag}:${tag} $DRY_RUN" - git push ${review_remote} ${tag}:${tag} $DRY_RUN + echo "git push ${review_remote} ${branch}:${branch} $DRY_RUN" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${branch}:${branch} $DRY_RUN && \ + echo "git push ${review_remote} ${tag}:${tag} $DRY_RUN" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag}:${tag} $DRY_RUN fi else - echo "git push ${remote} ${branch}:${branch} $DRY_RUN" - git push ${remote} ${branch}:${branch} $DRY_RUN - echo "git push ${remote} ${tag}:${tag} $DRY_RUN" - git push ${remote} ${tag}:${tag} $DRY_RUN + echo "git push ${remote} ${branch}:${branch} $DRY_RUN" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${branch}:${branch} $DRY_RUN && \ + echo "git push ${remote} ${tag}:${tag} $DRY_RUN" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag}:${tag} $DRY_RUN fi if [ $? != 0 ] ; then @@ -314,22 +313,21 @@ if [ $MANIFEST -eq 1 ]; then port=$(url_port "${url}") path=$(url_path "${url}") if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then - echo "git push ${review_remote} ${tag} && \\" - echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\" - echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\" - echo "git review --yes --topic=${branch}" - - git push ${review_remote} ${tag} $DRY_RUN && \ + echo "git push ${review_remote} ${tag}" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag} $DRY_RUN && \ + echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag}" && \ $DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch}" && \ $DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ - $DRY_RUN_CMD git review --yes --topic="${branch}" + echo "git review --yes --topic=${branch/\//.}" && \ + $DRY_RUN_CMD with_retries -d 45 -t 15 -k 5 5 git review --yes --topic="${branch/\//.}" else - echo git push --tags ${review_remote} ${branch} $DRY_RUN - git push --tags ${review_remote} ${branch} $DRY_RUN + echo git push --tags ${review_remote} ${branch} $DRY_RUN && \ + with_retries -d 45 -t 15 -k 5 5 git push --tags ${review_remote} ${branch} $DRY_RUN fi else - echo git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN - git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN + echo git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN && \ + with_retries -d 45 -t 15 -k 5 5 git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN fi if [ $? != 0 ] ; then diff --git a/build-tools/branching/push_tags.sh b/build-tools/branching/push_tags.sh index 246600f7..01ace224 100755 --- a/build-tools/branching/push_tags.sh +++ b/build-tools/branching/push_tags.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -182,10 +182,10 @@ for subgit in $SUBGITS; do echo "Pushing tag $tag in ${subgit}" if [ "${review_method}" == "gerrit" ] && [ $BYPASS_GERRIT -eq 0 ]; then echo "git push ${review_remote} ${tag}" - git push ${review_remote} ${tag} ${DRY_RUN} + with_retries -d 45 -t 15 -k 5 5 git push ${review_remote} ${tag} ${DRY_RUN} else echo "git push ${remote} ${tag}" - git push ${remote} ${tag} ${DRY_RUN} + with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag} ${DRY_RUN} fi if [ $? != 0 ] ; then @@ -246,7 +246,7 @@ if [ $MANIFEST -eq 1 ]; then echo "Pushing tag $tag in ${new_manifest_dir}" if [ "${review_method}" == "gerrit" ] && [ $BYPASS_GERRIT -eq 0 ]; then - git review + with_retries -d 45 -t 15 -k 5 5 git review if [ $? != 0 ] ; then echo_stderr "ERROR: Failed to create git review from ${new_manifest_dir}" exit 1 @@ -255,8 +255,10 @@ if [ $MANIFEST -eq 1 ]; then echo " cd ${new_manifest_dir}" echo " git push ${review_remote} ${tag}" else - git push ${remote} ${local_branch}:${remote_branch} ${DRY_RUN} - git push ${remote} ${tag}:${tag} ${DRY_RUN} + echo "git push ${remote} ${local_branch}:${remote_branch}" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${local_branch}:${remote_branch} ${DRY_RUN} && \ + echo "git push ${remote} ${tag}:${tag}" && \ + with_retries -d 45 -t 15 -k 5 5 git push ${remote} ${tag}:${tag} ${DRY_RUN} fi if [ $? != 0 ] ; then diff --git a/build-tools/git-repo-utils.sh b/build-tools/git-repo-utils.sh index e1b54bf7..395a343e 100644 --- a/build-tools/git-repo-utils.sh +++ b/build-tools/git-repo-utils.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -182,8 +182,7 @@ git_repo_review_remote () { if [ $? -ne 0 ]; then # Perhaps we need to run git review -s' and try again - # echo a blank line into git in case it prompts for a username - echo | git review -s > /dev/null || return 1 + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1 git config remote.gerrit.url > /dev/null || return 1 fi echo "gerrit" @@ -205,8 +204,7 @@ git_repo_review_url () { if [ $? -ne 0 ]; then # Perhaps we need to run git review -s' and try again - # echo a blank line into git in case it prompts for a username - echo | git review -s > /dev/null || return 1 + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1 git config remote.gerrit.url || return 1 fi else diff --git a/build-tools/git-utils.sh b/build-tools/git-utils.sh index a33f6a17..68319a42 100755 --- a/build-tools/git-utils.sh +++ b/build-tools/git-utils.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018 Wind River Systems, Inc. +# Copyright (c) 2018-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -11,10 +11,10 @@ # by repo manifests. # -echo_stderr () -{ - echo "$@" >&2 -} +GIT_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" + +source ${GIT_UTILS_DIR}/utils.sh + git_ctx_root_dir () { dirname "${MY_REPO}" @@ -562,7 +562,7 @@ git_review_url () { git config remote.gerrit.url if [ $? -ne 0 ]; then # Perhaps we need to run git review -s' and try again - git review -s > /dev/null || return 1 + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1 git config remote.gerrit.url fi else @@ -577,7 +577,7 @@ git_review_remote () { git config remote.gerrit.url > /dev/null if [ $? -ne 0 ]; then # Perhaps we need to run git review -s' and try again - git review -s > /dev/null || return 1 + with_retries -d 45 -t 15 -k 5 5 git review -s >&2 || return 1 git config remote.gerrit.url > /dev/null || return 1 fi echo "gerrit" diff --git a/build-tools/repo-utils.sh b/build-tools/repo-utils.sh index f3fdf54b..6cfb28c9 100644 --- a/build-tools/repo-utils.sh +++ b/build-tools/repo-utils.sh @@ -1,7 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020 Wind River Systems, Inc. +# Copyright (c) 2020-2021 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -10,16 +10,10 @@ # A collection of utilities relating to 'repo' # +REPO_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" -# -# Echo to stderr -# echo_stderr [any text you want] -# +source ${REPO_UTILS_DIR}/utils.sh -echo_stderr () -{ - echo "$@" >&2 -} # # Get the root dir of a repo managed repository diff --git a/build-tools/utils.sh b/build-tools/utils.sh new file mode 100644 index 00000000..52a9b97b --- /dev/null +++ b/build-tools/utils.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Copyright (c) 2019-2021 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Image and wheel build utility functions +# + +# +# Echo to stderr +# echo_stderr [any text you want] +# +echo_stderr () +{ + echo "$@" >&2 +} + + +# +# Function to call a command, with support for retries +# +# with_retries [] [...] +# +# Options: +# -d | --delay +# Wait given number of seconds between retries +# -t | --timeout +# Each iteration of the command runs under a timeout +# -k | --kill-timeout +# Each iteration of the command is killed violently +# if it doesn't exit voluntarily within the set time +# after the initial timeout signal. +# +function with_retries { + local delay=5 + local max_time=0 + local kill_time=0 + local to_cmd="" + + while [ $1 != "" ]; do + case "$1" in + -d | --delay) + delay=$2 + shift 2 + ;; + -t | --timeout) + max_time=$2 + shift 2 + ;; + -k | --kill-timeout) + kill_time=$2 + shift 2 + ;; + *) + break + ;; + esac + done + + local max_attempts=$1 + local cmd=$2 + shift 2 + + if [ ${max_time} -gt 0 ]; then + to_cmd="timeout " + if [ ${kill_time} -gt 0 ]; then + to_cmd+="--kill-after=${kill_time} " + fi + to_cmd+="${max_time} " + fi + + # Pop the first two arguments off the list, + # so we can pass additional args to the command safely + + local -i attempt=0 + local rc=0 + + while :; do + let attempt++ + + echo_stderr "Running: ${cmd} $@" + ${to_cmd} ${cmd} "$@" + rc=$? + if [ $rc -eq 0 ]; then + return 0 + fi + + if [ $rc -eq 124 ]; then + echo_stderr "Command (${cmd}) timed out, attempt ${attempt} of ${max_attempts}." + elif [ $rc -eq 137 ]; then + echo_stderr "Command (${cmd}) timed out and killed, attempt ${attempt} of ${max_attempts}." + else + echo_stderr "Command (${cmd}) failed, attempt ${attempt} of ${max_attempts}." + fi + + if [ ${attempt} -lt ${max_attempts} ]; then + echo_stderr "Waiting ${delay} seconds before retrying..." + sleep ${delay} + continue + else + echo_stderr "Max command attempts reached. Aborting..." + return 1 + fi + done +} +