diff --git a/functions b/functions index 6f2164a777..ebcae334e0 100644 --- a/functions +++ b/functions @@ -45,6 +45,37 @@ function short_source { # export it so child shells have access to the 'short_source' function also. export -f short_source +# Download a file from a URL +# +# Will check cache (in $FILES) or download given URL. +# +# Argument is the URL to the remote file +# +# Will echo the local path to the file as the output. Will die on +# failure to download. +# +# Files can be pre-cached for CI environments, see EXTRA_CACHE_URLS +# and tools/image_list.sh +function get_extra_file { + local file_url=$1 + + file_name=$(basename "$file_url") + if [[ $file_url != file* ]]; then + # If the file isn't cache, download it + if [[ ! -f $FILES/$file_name ]]; then + wget --progress=dot:giga -c $file_url -O $FILES/$file_name + if [[ $? -ne 0 ]]; then + die "$file_url could not be downloaded" + fi + fi + echo "$FILES/$file_name" + return + else + # just strip the file:// bit and that's the path to the file + echo $file_url | sed 's/$file:\/\///g' + fi +} + # Retrieve an image from a URL and upload into Glance. # Uses the following variables: diff --git a/lib/etcd3 b/lib/etcd3 index 6e32cb31ca..60e827add2 100644 --- a/lib/etcd3 +++ b/lib/etcd3 @@ -24,15 +24,9 @@ set +o xtrace # -------- # Set up default values for etcd -ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL:-https://github.com/coreos/etcd/releases/download} -ETCD_VERSION=${ETCD_VERSION:-v3.1.7} ETCD_DATA_DIR="$DATA_DIR/etcd" ETCD_SYSTEMD_SERVICE="devstack@etcd.service" ETCD_BIN_DIR="$DEST/bin" -ETCD_SHA256_AMD64="4fde194bbcd259401e2b5c462dfa579ee7f6af539f13f130b8f5b4f52e3b3c52" -# NOTE(sdague): etcd v3.1.7 doesn't have anything for these architectures, though 3.2.0 does. -ETCD_SHA256_ARM64="" -ETCD_SHA256_PPC64="" ETCD_PORT=2379 if is_ubuntu ; then @@ -95,37 +89,19 @@ function cleanup_etcd3 { function install_etcd3 { echo "Installing etcd" - # Make sure etcd3 downloads the correct architecture - if is_arch "x86_64"; then - ETCD_ARCH="amd64" - ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_AMD64} - elif is_arch "aarch64"; then - ETCD_ARCH="arm64" - ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_ARM64} - elif is_arch "ppc64le"; then - ETCD_ARCH="ppc64le" - ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64} - else - exit_distro_not_supported "invalid hardware type - $ETCD_ARCH" - fi - - ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH - # Create the necessary directories sudo mkdir -p $ETCD_BIN_DIR sudo mkdir -p $ETCD_DATA_DIR # Download and cache the etcd tgz for subsequent use + local etcd_file + etcd_file="$(get_extra_file $ETCD_DOWNLOAD_LOCATION)" if [ ! -f "$FILES/etcd-$ETCD_VERSION-linux-$ETCD_ARCH/etcd" ]; then - ETCD_DOWNLOAD_FILE=$ETCD_NAME.tar.gz - if [ ! -f "$FILES/$ETCD_DOWNLOAD_FILE" ]; then - wget $ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE -O $FILES/$ETCD_DOWNLOAD_FILE - fi - echo "${ETCD_SHA256} $FILES/${ETCD_DOWNLOAD_FILE}" > $FILES/etcd.sha256sum + echo "${ETCD_SHA256} $etcd_file" > $FILES/etcd.sha256sum # NOTE(sdague): this should go fatal if this fails sha256sum -c $FILES/etcd.sha256sum - tar xzvf $FILES/$ETCD_DOWNLOAD_FILE -C $FILES + tar xzvf $etcd_file -C $FILES sudo cp $FILES/$ETCD_NAME/etcd $ETCD_BIN_DIR/etcd fi if [ ! -f "$ETCD_BIN_DIR/etcd" ]; then diff --git a/stackrc b/stackrc index b123d8ac7c..0757d78775 100644 --- a/stackrc +++ b/stackrc @@ -732,6 +732,40 @@ if [[ "$DOWNLOAD_DEFAULT_IMAGES" == "True" ]]; then DOWNLOAD_DEFAULT_IMAGES=False fi +# This is a comma separated list of extra URLS to be listed for +# download by the tools/image_list.sh script. CI environments can +# pre-download these URLS and place them in $FILES. Later scripts can +# then use "get_extra_file " which will print out the path to the +# file; it will either be downloaded on demand or acquired from the +# cache if there. +EXTRA_CACHE_URLS="" + +# etcd3 defaults +ETCD_VERSION=${ETCD_VERSION:-v3.1.7} +ETCD_SHA256_AMD64="4fde194bbcd259401e2b5c462dfa579ee7f6af539f13f130b8f5b4f52e3b3c52" +# NOTE(sdague): etcd v3.1.7 doesn't have anything for these architectures, though 3.2.0 does. +ETCD_SHA256_ARM64="" +ETCD_SHA256_PPC64="" +# Make sure etcd3 downloads the correct architecture +if is_arch "x86_64"; then + ETCD_ARCH="amd64" + ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_AMD64} +elif is_arch "aarch64"; then + ETCD_ARCH="arm64" + ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_ARM64} +elif is_arch "ppc64le"; then + ETCD_ARCH="ppc64le" + ETCD_SHA256=${ETCD_SHA256:-$ETCD_SHA256_PPC64} +else + exit_distro_not_supported "invalid hardware type - $ETCD_ARCH" +fi +ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL:-https://github.com/coreos/etcd/releases/download} +ETCD_NAME=etcd-$ETCD_VERSION-linux-$ETCD_ARCH +ETCD_DOWNLOAD_FILE=$ETCD_NAME.tar.gz +ETCD_DOWNLOAD_LOCATION=$ETCD_DOWNLOAD_URL/$ETCD_VERSION/$ETCD_DOWNLOAD_FILE +# etcd is always required, so place it into list of pre-cached downloads +EXTRA_CACHE_URLS+=",$ETCD_DOWNLOAD_LOCATION" + # Detect duplicate values in IMAGE_URLS for image_url in ${IMAGE_URLS//,/ }; do if [ $(echo "$IMAGE_URLS" | grep -o -F "$image_url" | wc -l) -gt 1 ]; then diff --git a/tools/image_list.sh b/tools/image_list.sh index 29b93ed1d8..3a27c4acfd 100755 --- a/tools/image_list.sh +++ b/tools/image_list.sh @@ -1,5 +1,14 @@ #!/bin/bash +# Print out a list of image and other files to download for caching. +# This is mostly used by the OpenStack infrasturucture during daily +# image builds to save the large images to /opt/cache/files (see [1]) +# +# The two lists of URL's downloaded are the IMAGE_URLS and +# EXTRA_CACHE_URLS, which are setup in stackrc +# +# [1] project-config:nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos + # Keep track of the DevStack directory TOP_DIR=$(cd $(dirname "$0")/.. && pwd) @@ -31,12 +40,20 @@ for driver in $DRIVERS; do ALL_IMAGES+=$URLS done -# Make a nice list -echo $ALL_IMAGES | tr ',' '\n' | sort | uniq - # Sanity check - ensure we have a minimum number of images num=$(echo $ALL_IMAGES | tr ',' '\n' | sort | uniq | wc -l) if [[ "$num" -lt 4 ]]; then echo "ERROR: We only found $num images in $ALL_IMAGES, which can't be right." exit 1 fi + +# This is extra non-image files that we want pre-cached. This is kept +# in a separate list because devstack loops over the IMAGE_LIST to +# upload files glance and these aren't images. (This was a bit of an +# after-thought which is why the naming around this is very +# image-centric) +URLS=$(source $TOP_DIR/stackrc && echo $EXTRA_CACHE_URLS) +ALL_IMAGES+=$URLS + +# Make a nice combined list +echo $ALL_IMAGES | tr ',' '\n' | sort | uniq