starlingx/root
Code Issues Proposed changes

branching: better control over reviews & manifests

Browse Source 
* new option --safe-gerrit-host in branching scripts for specifying
gerrit hosts that are safe for pushing reviews to. Allows us to create
gerrit reviews for repos not hosted on opendev.org

* new option --manifest-file for overriding default.xml location, in
case the manifest <include>s a file whose name can't be deduced
automatically

* new option --dry-run in push_* scripts

Story: 2008998
Task: 42661

Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
Change-Id: I8e7395f7d64c5dfb8936f747d8bea65be22120fd
This commit is contained in:
Davlet Panech 2021-06-16 14:49:54 -04:00
parent 2090ecfbf3
commit dbc3f3c7c4
8 changed files with 188 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
build-tools/branching/create_branches_and_tags.sh
View File

@ -24,7 +24,10 @@ CREATE_BRANCHES_AND_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )"
source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh"
usage () {
echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --gitreview-default ] [ --manifest [ --lock-down | --soft-lock-down ] [ --default-revision ]]"
echo "create_branches_and_tags.sh --branch=<branch> [--tag=<tag>]"
echo " [ --remotes=<remotes> ] [ --projects=<projects> ]"
echo " [ --manifest [ --manifest-file=<file.xml> ] [ --lock-down | --soft-lock-down ] [ --default-revision ]]"
echo " [ --gitreview-default ] [ --safe-gerrit-host=<host> ]"
echo ""
echo "Create a branch and a tag in all listed projects, and all"
echo "projects hosted by all listed remotes. Lists are comma separated."
@ -43,9 +46,14 @@ usage () {
echo "If a gitreview-default is selected, then all branched projects"
echo "with a .gitreview file will have a defaultbranch entry added"
echo "or updated."
echo ""
echo "--manifest-file may be used to override the manifest file to be updated."
echo ""
echo "--safe-gerrit-host allows one to specify host names of gerrit servers"
echo "that are safe to push reviews to."
}
TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,help -n 'create_branches_and_tags.sh' -- "$@")
TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,manifest-file:,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,safe-gerrit-host:,help -n 'create_branches_and_tags.sh' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -65,6 +73,7 @@ manifest=""
new_manifest=""
repo_root_dir=""
safe_gerrit_hosts=()
while true ; do
case "$1" in
-h|--help) HELP=1 ; shift ;;
@ -73,15 +82,18 @@ while true ; do
--branch) branch=$2; shift 2;;
--tag) tag=$2; shift 2;;
--manifest) MANIFEST=1 ; shift ;;
--manifest-file) repo_set_manifest_file "$2" ; shift 2 ;;
--lock-down) LOCK_DOWN=2 ; shift ;;
--hard-lock-down) LOCK_DOWN=2 ; shift ;;
--soft-lock-down) LOCK_DOWN=1 ; shift ;;
--default-revision) SET_DEFAULT_REVISION=1 ; shift ;;
--gitreview-default) GITREVIEW_DEFAULT=1 ; shift ;;
--safe-gerrit-host) safe_gerrit_hosts+=("$2") ; shift 2 ;;
--) shift ; break ;;
*) usage; exit 1 ;;
esac
done
git_set_safe_gerrit_hosts "${safe_gerrit_hosts[@]}"
if [ $HELP -eq 1 ]; then
usage

15
build-tools/branching/create_tags.sh
View File

@ -23,7 +23,9 @@ CREATE_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${CREATE_TAGS_SH_DIR}/../git-repo-utils.sh"
usage () {
echo "create_tags.sh --tag=<tag> [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest [ --manifest-prefix <prefix> ] [ --lock-down | --soft-lock-down ] [ --default-revision ]]"
echo "create_tags.sh --tag=<tag> [ --remotes=<remotes> ] [ --projects=<projects> ]"
echo " [ --manifest [ --manifest-file=<manifest.xml> ] [ --manifest-prefix <prefix> ] [ --lock-down | --soft-lock-down ] [ --default-revision ]]"
echo " [ --safe-gerrit-host=<host> ]"
echo " "
echo "Create a git tag in all listed projects, and all projects"
echo "hosted by all listed remotes. Lists are comma separated."
@ -34,9 +36,14 @@ usage () {
echo "HEAD's sha set as the revision."
echo "If default-revision is selected, then the manifest default revision"
echo "will be set."
echo ""
echo "--manifest-file may be used to override the manifest file to be updated."
echo ""
echo "--safe-gerrit-host allows one to specify host names of gerrit servers"
echo "that are safe to push reviews to."
}
TEMP=$(getopt -o h --long remotes:,projects:,tag:,manifest,manifest-prefix:,lock-down,hard-lock-down,soft-lock-down,default-revision,help -n 'create_tags.sh' -- "$@")
TEMP=$(getopt -o h --long remotes:,projects:,tag:,manifest,manifest-file:,manifest-prefix:,lock-down,hard-lock-down,soft-lock-down,default-revision,safe-gerrit-host:,help -n 'create_tags.sh' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -55,6 +62,7 @@ new_manifest=""
manifest_prefix=""
repo_root_dir=""
safe_gerrit_hosts=()
while true ; do
case "$1" in
-h|--help) HELP=1 ; shift ;;
@ -62,15 +70,18 @@ while true ; do
--projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;;
--tag) tag=$2; shift 2;;
--manifest) MANIFEST=1 ; shift ;;
--manifest-file) repo_set_manifest_file "$2"; shift 2;;
--manifest-prefix) manifest_prefix=$2; shift 2;;
--lock-down) LOCK_DOWN=2 ; shift ;;
--hard-lock-down) LOCK_DOWN=2 ; shift ;;
--soft-lock-down) LOCK_DOWN=1 ; shift ;;
--default-revision) SET_DEFAULT_REVISION=1 ; shift ;;
--safe-gerrit-host) safe_gerrit_hosts+=("$2") ; shift 2 ;;
--) shift ; break ;;
*) usage; exit 1 ;;
esac
done
git_set_safe_gerrit_hosts "${safe_gerrit_hosts[@]}"
if [ $HELP -eq 1 ]; then
usage

67
build-tools/branching/push_branches_tags.sh
View File

@ -20,7 +20,11 @@ source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh"
source "${PUSH_BRANCHES_TAGS_SH_DIR}/../url_utils.sh"
usage () {
echo "push_branches_tags.sh --branch=<branch> [--tag=<tag>] [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest ] [ --bypass-gerrit ]"
echo "push_branches_tags.sh --branch=<branch> [--tag=<tag>]"
echo " [ --remotes=<remotes> ] [ --projects=<projects> ]"
echo " [ --manifest [ --manifest-file=<file.xml> ] ]"
echo " [ --bypass-gerrit] [--safe-gerrit-host=<host>]"
echo " [ --dry-run ]"
echo ""
echo "Push a pre-existing branch and tag into all listed projects, and all"
echo "projects hosted by all listed remotes. Lists are comma separated."
@ -30,9 +34,16 @@ usage () {
echo "prefix 'v' to the branch name."
echo ""
echo "A manifest push can also be requested.vision."
echo ""
echo "--manifest-file may be used to override the manifest file to be updated."
echo ""
echo "--safe-gerrit-host allows one to specify host names of gerrit servers"
echo "that are safe to push reviews to."
echo ""
echo "--dry-run will print out git push commands without executing them"
}
TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,bypass-gerrit,manifest,help -n 'push_branches_tags.sh' -- "$@")
TEMP=$(getopt -o h,n --long remotes:,projects:,branch:,tag:,bypass-gerrit,manifest,manifest-file:,safe-gerrit-host:,help,dry-run -n 'push_branches_tags.sh' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -42,6 +53,7 @@ eval set -- "$TEMP"
HELP=0
MANIFEST=0
BYPASS_GERRIT=0
DRY_RUN=
remotes=""
projects=""
branch=""
@ -49,19 +61,24 @@ tag=""
manifest=""
repo_root_dir=""
safe_gerrit_hosts=()
while true ; do
case "$1" in
-h|--help) HELP=1 ; shift ;;
-n|--dry-run) DRY_RUN="--dry-run" ; DRY_RUN_CMD=":" ; shift ;;
--bypass-gerrit) BYPASS_GERRIT=1 ; shift ;;
--remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;;
--projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;;
--branch) branch=$2; shift 2;;
--tag) tag=$2; shift 2;;
--manifest) MANIFEST=1 ; shift ;;
--manifest-file) repo_set_manifest_file "$2" ; shift 2;;
--safe-gerrit-host) safe_gerrit_hosts+=("$2") ; shift 2;;
--) shift ; break ;;
*) usage; exit 1 ;;
esac
done
git_set_safe_gerrit_hosts "${safe_gerrit_hosts[@]}"
if [ $HELP -eq 1 ]; then
usage
@ -143,8 +160,6 @@ if [ "$tag" == "" ]; then
fi
echo "Finding subgits"
SUBGITS=$(repo forall $projects -c 'echo '"$repo_root_dir"'/$REPO_PATH')
@ -208,18 +223,23 @@ for subgit in $SUBGITS; do
host=$(url_server "${url}")
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ]; then
git push ${review_remote} ${tag} && \
ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
git review --topic="${branch}"
if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then
echo "git push ${review_remote} ${tag} && \\"
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\"
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\"
echo "git review --topic=${branch}"
git push $DRY_RUN ${review_remote} ${tag} && \
$DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
$DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
$DRY_RUN_CMD git review --topic="${branch}"
else
echo "git push --tags ${review_remote} ${branch}"
git push --tags ${review_remote} ${branch}
echo "git push --tags ${review_remote} ${branch} $DRY_RUN"
git push --tags ${review_remote} ${branch} $DRY_RUN
fi
else
echo "git push --tags --set-upstream ${remote} ${branch}"
git push --tags --set-upstream ${remote} ${branch}
echo "git push --tags --set-upstream ${remote} ${branch} $DRY_RUN"
git push --tags --set-upstream ${remote} ${branch} $DRY_RUN
fi
if [ $? != 0 ] ; then
@ -289,16 +309,23 @@ if [ $MANIFEST -eq 1 ]; then
host=$(url_server "${url}")
port=$(url_port "${url}")
path=$(url_path "${url}")
if [ "${host}" == "review.opendev.org" ]; then
git push ${review_remote} ${tag} && \
ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
git review --yes --topic="${branch}"
if [ "${host}" == "review.opendev.org" ] || git_match_safe_gerrit_host "${host}" ; then
echo "git push ${review_remote} ${tag} && \\"
echo "ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \\"
echo "git config --local --replace-all branch.${branch}.merge refs/heads/${branch} && \\"
echo "git review --yes --topic=${branch}"
git push ${review_remote} ${tag} $DRY_RUN && \
$DRY_RUN_CMD ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \
$DRY_RUN_CMD git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \
$DRY_RUN_CMD git review --yes --topic="${branch}"
else
git push --tags ${review_remote} ${branch}
echo git push --tags ${review_remote} ${branch} $DRY_RUN
git push --tags ${review_remote} ${branch} $DRY_RUN
fi
else
git push --tags --set-upstream ${review_remote} ${branch}
echo git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN
git push --tags --set-upstream ${review_remote} ${branch} $DRY_RUN
fi
if [ $? != 0 ] ; then

27
build-tools/branching/push_tags.sh
View File

@ -19,15 +19,24 @@ PUSH_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source "${PUSH_TAGS_SH_DIR}/../git-repo-utils.sh"
usage () {
echo "push_tags.sh --tag=<tag> [ --remotes=<remotes> ] [ --projects=<projects> ] [ --manifest [--manifest-prefix <prefix>]] [ --bypass-gerrit ]"
echo "push_tags.sh --tag=<tag> [ --remotes=<remotes> ] [ --projects=<projects> ]"
echo " [ --manifest [ --manifest-file=<manifest.xml> ] [--manifest-prefix <prefix>]]"
echo " [ --bypass-gerrit ] [--safe-gerrit-host=<host>]"
echo " [ --dry-run ]"
echo " "
echo "Push a pre-existing git tag into all listed projects, and all projects"
echo "hosted by all listed remotes. Lists are comma separated."
echo ""
echo "A manifest push can also be requested."
echo ""
echo "--manifest-file may be used to override the manifest file to be updated."
echo ""
echo "--safe-gerrit-host allows one to specify host names of gerrit servers"
echo "that are safe to push reviews to."
}
TEMP=$(getopt -o h --long remotes:,projects:,tag:,manifest,manifest-prefix:,bypass-gerrit,help -n 'push_tags.sh' -- "$@")
TEMP=$(getopt -o h,n --long remotes:,projects:,tag:,manifest,manifest-file:,manifest-prefix:,bypass-gerrit,safe-gerrit-host:,help,dry-run -n 'push_tags.sh' -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
@ -35,6 +44,7 @@ fi
eval set -- "$TEMP"
HELP=0
DRY_RUN=
MANIFEST=0
BYPASS_GERRIT=0
remotes=""
@ -45,19 +55,24 @@ manifest_prefix=""
new_manifest=""
repo_root_dir=""
safe_gerrit_hosts=()
while true ; do
case "$1" in
-h|--help) HELP=1 ; shift ;;
-n|--dry-run) DRY_RUN="--dry-run" ; shift ;;
--bypass-gerrit) BYPASS_GERRIT=1 ; shift ;;
--remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;;
--projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;;
--tag) tag=$2; shift 2;;
--manifest) MANIFEST=1 ; shift ;;
--manifest-file) repo_set_manifest_file "$2"; shift 2;;
--manifest-prefix) manifest_prefix=$2; shift 2;;
--safe-gerrit-host) safe_gerrit_hosts+=("$2") ; shift 2 ;;
--) shift ; break ;;
*) usage; exit 1 ;;
esac
done
git_set_safe_gerrit_hosts "${safe_gerrit_hosts[@]}"
if [ $HELP -eq 1 ]; then
usage
@ -167,10 +182,10 @@ for subgit in $SUBGITS; do
echo "Pushing tag $tag in ${subgit}"
if [ "${review_method}" == "gerrit" ] && [ $BYPASS_GERRIT -eq 0 ]; then
echo "git push ${review_remote} ${tag}"
git push ${review_remote} ${tag}
git push ${review_remote} ${tag} ${DRY_RUN}
else
echo "git push ${remote} ${tag}"
git push ${remote} ${tag}
git push ${remote} ${tag} ${DRY_RUN}
fi
if [ $? != 0 ] ; then
@ -240,8 +255,8 @@ if [ $MANIFEST -eq 1 ]; then
echo " cd ${new_manifest_dir}"
echo " git push ${review_remote} ${tag}"
else
git push ${remote} ${local_branch}:${remote_branch}
git push ${remote} ${tag}:${tag}
git push ${remote} ${local_branch}:${remote_branch} ${DRY_RUN}
git push ${remote} ${tag}:${tag} ${DRY_RUN}
fi
if [ $? != 0 ] ; then

29
build-tools/git-repo-utils.sh
View File

@ -22,7 +22,7 @@ GIT_REPO_UTILS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"
source ${GIT_REPO_UTILS_DIR}/repo-utils.sh
source ${GIT_REPO_UTILS_DIR}/git-utils.sh
source ${GIT_REPO_UTILS_DIR}/url_utils.sh
#
# git_repo_rel_dir [<dir>]:
@ -117,26 +117,6 @@ git_repo_remote_url () {
git config remote.$remote.url
}
url_to_host () {
local URL="${1}"
# Strip protocol, path, user/pwd, port
echo "${URL}" | sed -e 's#^[^:]*://##' -e 's#/.*$##' -e 's#^[^@]*@##' -e 's#:.*$##'
}
host_to_domain () {
local host="${1}"
local elements=0
elements=$(echo "${host}" | sed 's#[^.]##g' | wc --chars)
if [ $elements -gt 2 ]; then
# strip lead element
echo "${host}" | sed 's#^[^.]*.##'
else
echo "${host}"
fi
}
git_repo_review_method () {
local DIR="${1:-${PWD}}"
local GIT_DIR=""
@ -174,6 +154,13 @@ git_repo_review_method () {
echo 'gerrit'
return 0
else
# review host is one of the globally-configured hosts that
# we know are safe
if git_match_safe_gerrit_host "${review_host}" ; then
echo 'gerrit'
return 0
fi
# Domains don't match. Not close enough to say gerrit is safe.
# Did someone forget to update .gitreview?
# Are we not pulling from the authoritative source?

54
build-tools/git-utils.sh
View File

@ -40,7 +40,6 @@ export GIT_LIST=$(git_list "$(git_ctx_root_dir)")
# as relative paths.
export GIT_LIST_REL=$(for p in $GIT_LIST; do echo .${p#$(git_ctx_root_dir)}; done)
#
# git_list_containing_branch <dir> <branch>:
# Return a list of git root directories found under <dir> and
@ -472,14 +471,61 @@ git_remote_branch () {
git config branch.${local_branch}.merge | sed 's#^refs/heads/##'
}
# Usage: git_set_safe_gerrit_hosts HOST1 HOST2...
# Set the host names that are safe to push reviews to
GIT_SAFE_GERRIT_HOSTS=()
git_set_safe_gerrit_hosts() {
GIT_SAFE_GERRIT_HOSTS=()
while [ "$#" -gt 0 ] ; do
GIT_SAFE_GERRIT_HOSTS+=("$1")
shift
done
}
# Usage: git_match_safe_gerrit_host HOSTNAME
# Return true if given host name is safe to push reviews to
# You have to call git_set_safe_gerrit_hosts() first
git_match_safe_gerrit_host() {
local review_host="$1"
local host
for host in "${GIT_SAFE_GERRIT_HOSTS[@]}" ; do
if [ "${review_host}" == "${host}" ]; then
return 0
fi
done
return 1
}
git_review_method () {
local url=""
local GIT_DIR
local url="" host=""
url=$(git_remote_url) || exit 1
if [[ "${url}" =~ "/git.starlingx.io/" || "${url}" =~ "/opendev.org/" ]]; then
echo 'gerrit'
else
echo 'default'
return 0
fi
GIT_DIR=$(git_root ${PWD}) || return 1
if [ ! -f ${GIT_DIR}/.gitreview ]; then
# No .gitreview file
echo 'default'
return 0
fi
if ! grep -q '\[gerrit\]' ${GIT_DIR}/.gitreview; then
# .gitreview file has no gerrit entry
echo 'default'
return 0
fi
review_host="$(grep host= ${GIT_DIR}/.gitreview | sed 's#^host=##' | head -n 1)"
if git_match_safe_gerrit_host "${review_host}" ; then
echo "gerrit"
return 0
fi
echo "default"
}
git_review_url () {

14
build-tools/repo-utils.sh
View File

@ -65,6 +65,11 @@ repo_root () {
# repo_manifest [<dir_path>]
#
REPO_MANIFEST_FILE=
repo_set_manifest_file() {
REPO_MANIFEST_FILE="$1"
}
repo_manifest () {
local query_dir="${1:-${PWD}}"
local root_dir=""
@ -75,6 +80,15 @@ repo_manifest () {
return 1
fi
if [[ -n "$REPO_MANIFEST_FILE" ]] ; then
if [[ "$REPO_MANIFEST_FILE" =~ ^/ ]] ; then
echo "$REPO_MANIFEST_FILE"
else
echo "${root_dir}/.repo/manifests/$REPO_MANIFEST_FILE"
fi
return 0
fi
repo_manifest="${root_dir}/.repo/manifest.xml"
# Depending on repo version, ${repo_manifest} is either a symlink to

20
build-tools/url_utils.sh
View File

@ -265,3 +265,23 @@ url_to_stx_mirror_url () {
echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH"
return 0
}
url_to_host () {
local URL="${1}"
# Strip protocol, path, user/pwd, port
echo "${URL}" | sed -e 's#^[^:]*://##' -e 's#/.*$##' -e 's#^[^@]*@##' -e 's#:.*$##'
}
host_to_domain () {
local host="${1}"
local elements=0
elements=$(echo "${host}" | sed 's#[^.]##g' | wc --chars)
if [ $elements -gt 2 ]; then
# strip lead element
echo "${host}" | sed 's#^[^.]*.##'
else
echo "${host}"
fi
}