docker-images: don't embed wheel tar in images

This patch fixes a problem introduced by [1]: docker image script
embeds the wheels tarball in the docker image via a COPY command; then
deletes it during the build. This makes docker images unnecessarily
large. This lets us use local tar files with the docker image(s) being
built.

Recently [2] a patch was merged that adds a web server to the Debian
build environment, allowing us to access local files over HTTP and
making the COPY step unnecessary in the Docker file.

This patch removes the explicit downloading of the wheels tarball and
the COPY directive, while passing the wheel tarball to Loci as a URL,
allowing Loci to download it during the build w/o contributing to the FS
layer size.

This reduces the size of stx-fm-rest-api image by ~220 MB. Other Loci
images should experience similar savings.

[1] https://review.opendev.org/c/starlingx/root/+/857505
[2] https://review.opendev.org/c/starlingx/tools/+/873010

TESTS
=================

* Build a loci image with wheels tarball passed as a URL
* Build a loci image with wheels tarball passed as a local file name

Story: 2010294
Task: 47343

Change-Id: I6dca6f95b0845e549ad35c9b6e95aa58c86fc774
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>

build-tools/build-docker-images/build-stx-images.sh

@@ -112,6 +112,12 @@ function is_in {
     return 1
 }
 
+function starts_with {
+    local str="$1"
+    local prefix="$2"
+    [[ "${str#$prefix}" != "$str" ]]
+}
+
 function is_empty {
     test $# -eq 0
 }
@@ -121,20 +127,33 @@ function url_basename {
     echo "$1" | sed -r -e 's/[?#].*//' -e 's#.*/##'
 }
 
-# Usage: download $URL $OUTPUT_FILE
+function local_path_to_url {
-function download_file {
+    local path="$1"
-    local url="$1"
+
-    local out_file="$2"
+    local abs_path
-    if echo "$url" | grep -E -q -e '^https?:' -e '^ftp:' ; then
+    abs_path="$(readlink -f "$path")" || exit 1
-        \rm -f "$out_file.tmp" || return 1
+
-        with_retries 5 wget -O "$out_file.tmp" "$url" || return 1
+    local repo_root
-        \mv -f "$out_file.tmp" "$out_file" || exit 1
+    repo_root="$(readlink -e "$MY_REPO_ROOT_DIR")" || exit 1
+
+    local workspace_root
+    workspace_root="$(readlink -e "$MY_WORKSPACE")" || exit 1
+
+    local dflt_port
+    if starts_with "$abs_path" "$repo_root" ; then
+        dflt_port="8089"
+    elif starts_with "$abs_path" "$workspace" ; then
+        dflt_port="8088"
     else
-        local src_file
+        echo "ERROR: $path: path must start with \$MY_REPO_ROOT_DIR or \$MY_WORKSPACE" >&2
-        src_file="$(echo "$url" | sed -e 's#^file:/+##')"
+        exit 1
-        \cp -a "$src_file" "$out_file" || return 1
+    fi
+
+    if [[ -n "$BUILDER_FILES_URL" ]] ; then
+        echo "${BUILDER_FILES_URL}${path}"
+    else
+        echo "http://${HOSTNAME}:${dflt_port}${path}"
     fi
-    return 0
 }
 
 #
@@ -241,16 +260,6 @@ function patch_loci {
     \rm -rf "${WORKDIR}/loci/stx-wheels/"* || exit 1
 }
 
-function download_loci_wheels {
-    local url="$1"
-    out_file="${WORKDIR}/loci/stx-wheels/$(url_basename "$url")"
-    if [[ ! -f "$out_file" ]] ; then
-        echo "Downloading $url => $out_file" >&2
-        download_file "$url" "$out_file" || return 1
-    fi
-    return 0
-}
-
 function update_image_record {
     # Update the image record file with a new/updated entry
     local LABEL=$1
@@ -473,11 +482,7 @@ function build_image_loci {
             return 1
         fi
 
-        if [[ -n "$BUILDER_FILES_URL" ]] ; then
+        PROJECT_REPO="$(local_path_to_url "${CLONE_DIR}")" || exit 1
-            PROJECT_REPO="$BUILDER_FILES_URL/${CLONE_DIR}"
-        else
-            PROJECT_REPO="http://${HOSTNAME}:8088/${CLONE_DIR}"
-        fi
     fi
 
     local -a BUILD_ARGS=
@@ -487,14 +492,10 @@ function build_image_loci {
 
     if [ "${PYTHON3}" == "no" ] ; then
         echo "Python2 service ${LABEL}"
-        download_loci_wheels "$WHEELS_PY2" || exit 1
+        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
-        #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
-        BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS_PY2")")
     else
         echo "Python3 service ${LABEL}"
-        download_loci_wheels "$WHEELS" || exit 1
+        BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
-        #BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
-        BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS")")
     fi
 
     if [ ! -z "$HTTP_PROXY" ]; then
@@ -981,12 +982,16 @@ fi
 for var in WHEELS WHEELS_PY2 ; do
     # skip empty vars
     [[ -n "${!var}" ]] || continue
-    # skip network urls
+    # http(s) urls are supported by Loci directly -- skip
-    echo "${!var}" | grep -E -q -e '^https?:' -e '^ftp:' && continue
+    # See https://github.com/openstack/loci/blob/efccd0a853879ac6af6066eda09792d0d3afe9c0/scripts/fetch_wheels.py#L170
+    echo "${!var}" | grep -E -q -e '^https?:' && continue
     # remove file:/ prefix if any
     declare "$var=$(echo "${!var}" | sed -r 's#^file:/+##')"
     # resolve it to an absolute path
     declare "$var=$(readlink -f "${!var}")" || exit 1
+    # convert it to a local URL
+    url="$(local_path_to_url "${!var}")" || exit 1
+    declare "$var=$url"
 done
 
 # Find the directives files

build-tools/build-docker-images/loci/patches/0001-starlingx-enable-disable-package-repos.patch

@@ -1,39 +1,27 @@
-From 644ac10c8877444c540aac36111e59c65c47ce59 Mon Sep 17 00:00:00 2001
+From 7462c9467cd0a1e98ced03517646a4e00f65ddc3 Mon Sep 17 00:00:00 2001
 From: Davlet Panech <davlet.panech@windriver.com>
 Date: Thu, 8 Sep 2022 21:04:55 +0000
-Subject: [PATCH 1/2] starlingx: wheels from filesystem + pkg repos
+Subject: [PATCH] starlingx: enable/disable package repos
 
-- Allow WHEELS to be a local file path, rather than URL
+Dockerfile: new parameter DIST_REPOS that allows one to
-- Dockerfile: new parameter DIST_REPOS that allows one to
+enable/disable package repos when building
-  enable/disable package repos when building
 
 Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
 ---
- .gitignore                         |   1 +
+ Dockerfile                         |   5 +-
- Dockerfile                         |   6 +-
+ stx-scripts/install.sh             |  11 +++
- stx-scripts/cleanup.sh             |   6 ++
- stx-scripts/install.sh             |  14 ++++
  stx-scripts/setup-package-repos.sh | 126 +++++++++++++++++++++++++++++
  stx-wheels/.keep                   |   0
- 6 files changed, 152 insertions(+), 1 deletion(-)
+ 4 files changed, 141 insertions(+), 1 deletion(-)
- create mode 100644 .gitignore
- create mode 100755 stx-scripts/cleanup.sh
  create mode 100755 stx-scripts/install.sh
  create mode 100755 stx-scripts/setup-package-repos.sh
  create mode 100644 stx-wheels/.keep
 
-diff --git a/.gitignore b/.gitignore
-new file mode 100644
-index 0000000..4b5032a
---- /dev/null
-+++ b/.gitignore
-@@ -0,0 +1 @@
-+stx-wheels/[^.]*
 diff --git a/Dockerfile b/Dockerfile
-index 3a026a3..145d284 100644
+index 3a026a3..3baea6c 100644
 --- a/Dockerfile
 +++ b/Dockerfile
-@@ -32,4 +32,8 @@ ARG SPICE_REF=${SPICE_REF:-spice-html5-0.1.6}
+@@ -32,4 +32,7 @@ ARG SPICE_REF=${SPICE_REF:-spice-html5-0.1.6}
  COPY scripts /opt/loci/scripts
  ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/
  
@@ -41,26 +29,13 @@ index 3a026a3..145d284 100644
 +#RUN /opt/loci/scripts/install.sh
 +ARG DIST_REPOS
 +COPY stx-scripts /opt/loci/stx-scripts
-+COPY stx-wheels  /opt/loci/stx-wheels
 +RUN /opt/loci/stx-scripts/install.sh
-diff --git a/stx-scripts/cleanup.sh b/stx-scripts/cleanup.sh
-new file mode 100755
-index 0000000..6ac890f
---- /dev/null
-+++ b/stx-scripts/cleanup.sh
-@@ -0,0 +1,6 @@
-+#!/bin/bash
-+
-+set -ex
-+
-+rm -rf /opt/loci/stx-wheels/*
-+
 diff --git a/stx-scripts/install.sh b/stx-scripts/install.sh
 new file mode 100755
-index 0000000..033bdb9
+index 0000000..da11b75
 --- /dev/null
 +++ b/stx-scripts/install.sh
-@@ -0,0 +1,14 @@
+@@ -0,0 +1,11 @@
 +#!/bin/bash
 +
 +set -ex
@@ -72,9 +47,6 @@ index 0000000..033bdb9
 +
 +# run Loci installer
 +"$LOCI_DIR/scripts/install.sh" "$@"
-+
-+# delete wheel tarball etc
-+"$LOCI_DIR/stx-scripts/cleanup.sh"
 diff --git a/stx-scripts/setup-package-repos.sh b/stx-scripts/setup-package-repos.sh
 new file mode 100755
 index 0000000..dd43612