From 921cb526ffd9fe9d6dddcf7253bf0812d7299350 Mon Sep 17 00:00:00 2001 From: Don Penney Date: Wed, 12 Feb 2020 14:51:52 -0500 Subject: [PATCH] Add tag-management utility This commit introduces a tag management utility for images to allow retagging of specific images. This allows the StarlingX ansible playbooks and packaged charts to use a static image tag rather than the moving stable-latest tag. This utility can only be run by a StarlingX build team member with publishing capability on the docker hub, to allow for updated tags to be available publicly, or as part of a CENGN build job. Change-Id: I7659695c701ccf3bfe8255b24ad0b1896b59cc23 Partial-Bug: 1854869 Signed-off-by: Don Penney --- .../tag-management/image-tags-3.0.yaml | 38 ++++ .../tag-management/image-tags-4.0.yaml | 48 +++++ .../tag-management/retag-images.sh | 192 ++++++++++++++++++ 3 files changed, 278 insertions(+) create mode 100644 build-tools/build-docker-images/tag-management/image-tags-3.0.yaml create mode 100644 build-tools/build-docker-images/tag-management/image-tags-4.0.yaml create mode 100755 build-tools/build-docker-images/tag-management/retag-images.sh diff --git a/build-tools/build-docker-images/tag-management/image-tags-3.0.yaml b/build-tools/build-docker-images/tag-management/image-tags-3.0.yaml new file mode 100644 index 00000000..9a8850bf --- /dev/null +++ b/build-tools/build-docker-images/tag-management/image-tags-3.0.yaml @@ -0,0 +1,38 @@ +# yamllint disable rule:line-length +--- +images: + - name: docker.io/starlingx/k8s-cni-sriov + src_build_tag: master-centos-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 + # Tag determined based on release tag associated with upstream commit + tag: stx.3.0-v2.2 + - name: docker.io/starlingx/k8s-plugins-sriov-network-device + src_build_tag: master-centos-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 + # Tag determined based on release tag associated with upstream commit + tag: stx.3.0-v3.1 + - name: docker.io/starlingx/intel-fpga-admissionwebhook + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.3.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-fpga-initcontainer + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.3.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-fpga-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.3.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-gpu-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.3.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-qat-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/d9d92a6997f939cb1ec41813dbf2d72d0b80d79c + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.3.0-v0.11.0-109-gc48c502 diff --git a/build-tools/build-docker-images/tag-management/image-tags-4.0.yaml b/build-tools/build-docker-images/tag-management/image-tags-4.0.yaml new file mode 100644 index 00000000..f8421e55 --- /dev/null +++ b/build-tools/build-docker-images/tag-management/image-tags-4.0.yaml @@ -0,0 +1,48 @@ +# yamllint disable rule:line-length +--- +images: + - name: docker.io/starlingx/k8s-cni-sriov + src_build_tag: master-centos-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 + # Tag determined based on release tag associated with upstream commit + tag: stx.4.0-v2.2 + - name: docker.io/starlingx/k8s-plugins-sriov-network-device + src_build_tag: master-centos-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 + # Tag determined based on release tag associated with upstream commit + tag: stx.4.0-v3.1 + - name: docker.io/starlingx/intel-fpga-admissionwebhook + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.4.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-fpga-initcontainer + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.4.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-fpga-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.4.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-gpu-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/5f72ddb26a38d41fef919060585daaafae677433 + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.4.0-v0.11.0-103-g4f28657 + - name: docker.io/starlingx/intel-qat-plugin + src_build_tag: master-distroless-stable-20191203T153530Z.0 + src_ref: https://opendev.org/starlingx/integ/commit/d9d92a6997f939cb1ec41813dbf2d72d0b80d79c + # Version determined by running 'git describe --tags' in clone of upstream repo + tag: stx.4.0-v0.11.0-109-gc48c502 + - name: docker.io/starlingx/stx-oidc-client + src_build_tag: master-centos-stable-20200204T162546Z.0 + src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/a63597fc038deac9fa20e35af61cbed73d39fd10 + # StarlingX app. Setting version to v1.0.0 + tag: stx.4.0-v1.0.0 + - name: docker.io/starlingx/dex + src_build_tag: master-centos-stable-20200204T162546Z.0 + src_ref: https://opendev.org/starlingx/oidc-auth-armada-app/commit/5d6701bdf214e77f460f2e3dd2b6f7d3186830c8 + # Tag determined based on release tag associated with upstream commit + tag: stx.4.0-v2.14.0-1 diff --git a/build-tools/build-docker-images/tag-management/retag-images.sh b/build-tools/build-docker-images/tag-management/retag-images.sh new file mode 100755 index 00000000..fddcd21e --- /dev/null +++ b/build-tools/build-docker-images/tag-management/retag-images.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This utility retags images, as configured by yaml files, formatted as: +# +# images: +# - name: docker.io/starlingx/k8s-cni-sriov +# src_build_tag: master-centos-stable-20191203T153530Z.0 +# src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 +# tag: stx.3.0-v2.2 +# - name: docker.io/starlingx/k8s-plugins-sriov-network-device +# src_build_tag: master-centos-stable-20191203T153530Z.0 +# src_ref: https://opendev.org/starlingx/integ/commit/dac417bd31ed36d455e94db4aabe5916367654d4 +# tag: stx.3.0-v3.1 +# + +declare RUNCMD= +declare -a REPUSH + +function usage { + cat >&2 <: Repush a specific image, even if the tag already exists + +EOF +} + +function is_in { + local search=$1 + shift + + for v in $*; do + if [ "${search}" = "${v}" ]; then + return 0 + fi + done + return 1 +} + +function parse_image_yaml { +python -c ' +import sys +import yaml + +for fname in sys.argv[1:]: + with open(fname) as f: + imgs = yaml.load_all(f, Loader=yaml.FullLoader) + for entry in imgs: + for img in entry.get("images"): + print ("%s|%s|%s|%s" % ( + img.get("name"), + img.get("tag"), + img.get("src_build_tag"), + img.get("src_ref"))) +' ${@} +} + +function get_tags_from_docker_hub { + local url=$1 + + curl -k -sSL -X GET ${url} | python -c ' +import sys, json +y=json.loads(sys.stdin.read()) +if y and y.get("next"): + print("next=%s" % y.get("next")) +if y and y.get("results"): + for res in y.get("results"): + if res.get("name"): + print("tag=%s" % res.get("name")) +' | while IFS='=' read key value; do + if [ "${key}" = "next" ]; then + get_tags_from_docker_hub ${value} + else + echo "${key}=${value}" + fi + done +} + +function retag_and_push_image { + local name=$1 + local src_tag=$2 + local new_tag=$3 + + # Break down the name into components + local docker_registry=${name%%/*} + local image=${name/${docker_registry}\/} + local repository_name=${image%/*} + local label=${image/*\/} + + if is_in $(basename $label) ${REPUSH[@]}; then + echo "Skipping existence check for ${name}" + else + if [ "${docker_registry}" = "docker.io" ]; then + get_tags_from_docker_hub https://registry.hub.docker.com/v2/repositories/${image}/tags \ + | grep -q "^tag=${new_tag}$" + if [ $? -eq 0 ]; then + # Already exists + echo "Image tag exists: ${name}:${new_tag}" + return 0 + fi + else + curl -k -sSL -X GET https://${docker_registry}/v2/${image}/tags/list \ + | python -c ' +import sys, yaml, json, re +y=yaml.load(sys.stdin.read(), Loader=yaml.FullLoader) +RC=1 +if y and sys.argv[1] in [img for img in y.get("tags")]: + RC=0 +sys.exit(RC) +' ${new_tag} + if [ $? -eq 0 ]; then + # Already exists + echo "Image tag exists: ${name}:${new_tag}" + return 0 + fi + fi + fi + + ${RUNCMD} docker image pull ${name}:${src_tag} + if [ $? -ne 0 ]; then + echo "Failed to pull ${name}:${src_tag}" >&2 + return 1 + fi + + ${RUNCMD} docker tag ${name}:${src_tag} ${name}:${new_tag} + if [ $? -ne 0 ]; then + echo "Failed to retag ${name}:${src_tag} as ${name}:${new_tag}" >&2 + return 1 + fi + + ${RUNCMD} docker push ${name}:${new_tag} + if [ $? -ne 0 ]; then + echo "Failed to push ${name}:${new_tag}" >&2 + return 1 + fi + + echo "Pushed: ${name}:${new_tag}" + + ${RUNCMD} docker image rm ${name}:${src_tag} ${name}:${new_tag} + if [ $? -ne 0 ]; then + echo "Failed to rm images ${name}:${src_tag} ${name}:${new_tag}" >&2 + fi + + return 0 +} + +OPTS=$(getopt -o h,c -l help,repush:,dryrun -- "$@") +if [ $? -ne 0 ]; then + usage + exit 1 +fi + +eval set -- "${OPTS}" + +while true; do + case $1 in + --) + # End of getopt arguments + shift + break + ;; + --repush) + # Read comma-separated values into array + REPUSH+=(${2//,/ }) + shift 2 + ;; + --dryrun) + RUNCMD=echo + shift + ;; + -h | --help ) + usage + exit 1 + ;; + *) + usage + exit 1 + ;; + esac +done + +parse_image_yaml ${@} | while IFS='|' read name new_tag src_tag src_ref; do + retag_and_push_image ${name} ${src_tag} ${new_tag} +done +