From 748a8b42c9d4f1789d8cace05d6e04c698232bea Mon Sep 17 00:00:00 2001
From: Vitaly Parakhin <vparakhin@mirantis.com>
Date: Fri, 15 May 2015 18:20:07 +0300
Subject: [PATCH] Add fuel-package-updates integration

---
 config/common.cfg | 11 +++++----
 deb-mirror        |  9 +++++++-
 fuel-createmirror | 58 +++++++++++++++++++++++++++++++++++++++--------
 util/rsync.sh     |  6 ++---
 4 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/config/common.cfg b/config/common.cfg
index 1529f17..50d098c 100644
--- a/config/common.cfg
+++ b/config/common.cfg
@@ -7,8 +7,11 @@ MIRROR_ROOT="/var/www/nailgun"
 LOG_ROOT="/var/log/mirror-sync"
 
 # In case of working behind http proxy uncomment the following parameters
-# RSYNC_PROXY=http://my-proxy-address:8080
-# http_proxy=$RSYNC_PROXY
+# Note that your proxy configuration must allow proxying to port 873
+# RSYNC_PROXY=username:password@host:port
+# export RSYNC_PROXY
+# http_proxy=http://username:password@host:port/
+# export http_proxy
 
 # Attempt to autodetect current Fuel version
 if hash fuel 2>/dev/null; then
@@ -22,8 +25,8 @@ if hash fuel 2>/dev/null; then
     # On Fuel node we will be using dockerized Ubuntu for dependencies resolving.
     # Assuming that Docker is up and running.
     DOCKER_MODE=true
-    FUEL_SERVER=`grep ipaddress: /etc/fuel/astute.yaml | awk '{print $2}'`
-    FUEL_ADMIN_PASS=`grep -A 2 FUEL_ACCESS  /etc/fuel/astute.yaml | grep password | awk '{print $2}'`
+    FUEL_SERVER=`grep '"ipaddress":' /etc/fuel/astute.yaml | awk '{print $2}' | sed 's/"//g'`
+    FUEL_ADMIN_PASS=`grep -A 2 FUEL_ACCESS  /etc/fuel/astute.yaml | grep password | awk '{print $2}' | sed 's/"//g'`
     FULL_RELEASE=`fuel --fuel-version 2>&1 | grep -A 1 release_versions | tail -1 | cut -d":" -f1`
 else
     # working outside Fuel master node, using defaults
diff --git a/deb-mirror b/deb-mirror
index 1e96b1a..2c4367c 100755
--- a/deb-mirror
+++ b/deb-mirror
@@ -89,6 +89,12 @@ mkdir -p $LOCAL_DIR/pool
 packages_pool_files=()
 sources_pool_files=()
 
+if rsync_file_exists "."; then
+    info "Upstream mirror $UPSTREAM supports rsync protocol"
+else
+    fatal "Upstream mirror $UPSTREAM does not support rsync protocol, aborting"
+fi
+
 debug_job_start "Checking if upstream mirror update is in progress..."
 while rsync_file_exists "Archive-Update-in-Progress*"; do
     info "'Archive-Update-in-Progress' file found on upstream mirror. Sleeping for 20 seconds"
@@ -476,6 +482,7 @@ if [[ "$DOCKER_MODE" = "true" ]]; then
         # docker pull ubuntu:latest
         docker -D run -d --name=fuel-createmirror --net=host -a stdout -a stderr -t \
                -e UBUNTU_RELEASE=$UBUNTU_RELEASE -e UBUNTU_NETBOOT_FLAVOR=$UBUNTU_NETBOOT_FLAVOR \
+               -e RSYNC_PROXY=$RSYNC_PROXY -e http_proxy=$http_proxy \
                -e UBUNTU_ARCH=$UBUNTU_ARCH -e FUEL_BRANCH=$FUEL_BRANCH -e BINROOT=$BINROOT \
                -e apt_altstate=$apt_altstate -v $BINROOT:$BINROOT:rw -v $apt_altstate:$apt_altstate:rw ubuntu:latest
         dockerctl shell fuel-createmirror $BINROOT/util/partial_ubuntu.sh || fatal "Cannot calculate list of dependencies"
@@ -502,7 +509,7 @@ fi # "$PARTIAL_UPSTREAM" = "1"
 
 info "Downloading pool files"
 rsync --verbose --out-format="%i %n" --stats \
-    --recursive --perms --links --times --hard-links --sparse --safe-links \
+    --recursive --perms --copy-links --times --hard-links --sparse --safe-links \
     --exclude=".tmp/" --exclude=".temp/" --exclude=".~tmp~/" \
     --files-from="$files_to_dl_list" \
     --bwlimit=5192 \
diff --git a/fuel-createmirror b/fuel-createmirror
index e663b29..3ac61aa 100755
--- a/fuel-createmirror
+++ b/fuel-createmirror
@@ -56,19 +56,47 @@ EOF
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-print_repositories() {
-    echo -e " * INFO: Go to Fuel UI, choose your cluster and go to the 'Settings' tab"
-    echo " * INFO: Set the following repositories:"; echo
-    if [ "$2" == "full" ]; then
+print_repositories_ubuntu() {
+    echo -e " * INFO: In order to setup these repositories MANUALLY, you should"
+    echo -e "         go to Fuel UI, choose your cluster and go to the 'Settings' tab"
+    if [ "$PARTIAL_UPSTREAM" == "0" ]; then
+	echo -e "         Replace the URI value for the following repositories:"
 	for dist in "${DISTs[@]}"; do
-	    echo "   $dist deb http://$FUEL_SERVER:8080/$1 $dist ${DIST_COMPONENTs[$dist]}"
+	    echo -e "         Repository \"$dist\" URI=\"deb http://$FUEL_SERVER:8080/$1 $dist ${DIST_COMPONENTs[$dist]}\""
 	done
     else
-	echo "   ${DISTs[0]} deb http://$FUEL_SERVER:8080/$1 ${DISTs[0]} main"
+	echo -e "         REMOVE all repositories named \"ubuntu-*\" and change URI for the following one:"
+	echo
+	echo -e "         Repository \"ubuntu\" new URI=\"deb http://$FUEL_SERVER:8080/$1 ${DISTs[0]} main\""
     fi
     echo
 }
 
+add_repositories_to_nailgun() {
+# parameters:
+# $1 - release_id for fuel env
+# $2 - distro name in fuel-package-updates format
+# $3 - directory name of local repository
+    echo " * INFO: Attempting to add created repositories to Nailgun..."
+    # find envs with status "new" and with given release_id
+    local release_id=$1
+    local extraopts=""
+    [ "$PARTIAL_UPSTREAM" == "1" ] && extraopts=" --clear-upstream-repos "
+    envs=`fuel env 2>&1 | grep -w new | awk -v release_id=$release_id -F'|' '$5 == release_id {print $1}'`
+    for env in ${envs}; do
+        fuel-package-updates -d $2 -r $FULL_RELEASE --no-download --apply --make-default \
+        -s $FUEL_SERVER -p "$FUEL_ADMIN_PASS" -b http://$FUEL_SERVER:8080/$3 -e $env $extraopts 2>/dev/null
+	EC_FPU=$?
+	if [[ "$EC_FPU" == "0" ]]; then
+	    echo " * INFO: environment id=$env updated successfully, no manual actions is required"
+	else
+	    echo " * WARNING: Failed to add repositories for environment id=$env to Nailgun, please add them MANUALLY"
+	    EC_ADD=1
+	fi
+    done
+    [ "$EC_ADD" == "1" ] && print_repositories_ubuntu $3
+}
+
 BINROOT=$(dirname `readlink -f "$0"`)
 
 if [[ ( "$1" == "--help" ) ||  ( "$1" == "-h" ) ]]; then 
@@ -97,7 +125,11 @@ if [[ "$1" != "ubuntu" ]]; then
     if [[ "$EC_MOS" == "0" ]]; then
 	. $BINROOT/config/mos-ubuntu-updatesonly.cfg
 	echo " * INFO: MOS mirror was created at: $LOCAL_DIR"
-	print_repositories ${LOCAL_DIR##*/} full
+	if [[ "$DOCKER_MODE" == "true" ]]; then
+	    add_repositories_to_nailgun 2 ubuntu ${LOCAL_DIR##*/}
+	else
+	    print_repositories_ubuntu ${LOCAL_DIR##*/}
+	fi
     else
 	echo " * FATAL: Creation of MOS mirror FAILED, check logs at $LOG_ROOT"
 	EC=1
@@ -108,10 +140,18 @@ if [[ "$1" != "mos" ]]; then
 	. $BINROOT/config/ubuntu.cfg
 	if [[ $PARTIAL_UPSTREAM = "1" ]]; then
 	    echo " * INFO: Ubuntu partial mirror was created at: $PARTIAL_UPSTREAM_PATH"
-	    print_repositories ${PARTIAL_UPSTREAM_PATH##*/}
+	    if [[ "$DOCKER_MODE" == "true" ]]; then
+		add_repositories_to_nailgun 2 ubuntu-baseos ${PARTIAL_UPSTREAM_PATH##*/}
+	    else
+		print_repositories_ubuntu ${PARTIAL_UPSTREAM_PATH##*/}
+	    fi
 	else
 	    echo " * INFO: Ubuntu mirror was created at: $LOCAL_DIR"
-	    print_repositories ${LOCAL_DIR##*/} full
+	    if [[ "$DOCKER_MODE" == "true" ]]; then
+		add_repositories_to_nailgun 2 ubuntu-baseos ${LOCAL_DIR##*/}
+	    else
+		print_repositories_ubuntu ${LOCAL_DIR##*/}
+	    fi
 	fi
     else
 	echo " * FATAL: Creation of Ubuntu mirror FAILED, check logs at $LOG_ROOT"
diff --git a/util/rsync.sh b/util/rsync.sh
index 40a4fab..303a590 100644
--- a/util/rsync.sh
+++ b/util/rsync.sh
@@ -16,7 +16,7 @@ rsync_ls()
 
     files=`/usr/bin/rsync --no-motd --list-only \
         --relative --recursive --no-implied-dirs \
-        --perms --links --times --hard-links --sparse --safe-links \
+        --perms --copy-links --times --hard-links --sparse --safe-links \
         "${UPSTREAM}::${UPSTREAM_DIR}/$mask" | \
         grep -v "^d" |  sed -e "s/->.*//g" | awk '{$1=$2=$3=$4=""}1'`
 
@@ -41,7 +41,7 @@ fetch()
     [[ -d "$dst_dir" ]] || mkdir -p "$dst_dir"
 
     debug_job_start "Fetching '$src_path' to '$dst_path' with params '${opt_args[@]}'"
-    /usr/bin/rsync --no-motd --perms --links --times --hard-links --sparse --safe-links \
+    /usr/bin/rsync --no-motd --perms --copy-links --times --hard-links --sparse --safe-links \
         ${opt_args[@]} \
         "${UPSTREAM}::${UPSTREAM_DIR}/$src_path" "$dst_path"
     local rsync_ec="$?"
@@ -65,7 +65,7 @@ fetch_all()
     rsync_out=` echo $* | tr ' ' '\n' | \
         rsync --no-motd --relative --out-format='%n' --files-from=- \
         --no-implied-dirs --no-motd \
-        --perms --links --times --hard-links --sparse \
+        --perms --copy-links --times --hard-links --sparse \
         "${UPSTREAM}::${UPSTREAM_DIR}/" "$root" 2> /dev/null`
     for line in $rsync_out; do
         debug "Fetched file $LOCAL_DIR/$line"