Merge "docker-images: add Loci patches"

This commit is contained in:
Zuul 2022-09-14 16:20:58 +00:00 committed by Gerrit Code Review
commit 9bfca3bcf6
8 changed files with 358 additions and 191 deletions

View File

@ -0,0 +1,8 @@
loci/patches:
- loci/patches/0001-starlingx-wheels-from-filesystem-pkg-repos.patch : this patch
makes docker images very large when COPY'ing a local wheels tarball.
- Remove all patches; or re-work & merge upstream & then switch to an upstream loci.
They may not apply to Loci/master as is; and they have problems as well as
some STX-specific functionality.

View File

@ -229,47 +229,11 @@ function get_loci {
}
function patch_loci {
# Loci contains a Docker file, which runs a script that installs
# Python modules etc based on build parameters. We replace the call
# to Loci's install script with our own, so that we can perform
# additional actions within the Dockerfile before Loci does its thing.
#
# loci/ <-- clone of Loci git repo
# Dockerfile
# ...
# Dockerfile.stx <-- create this by replacing "RUN..." in Dockerfile
# with the contents of loci/docker/Dockerfile.part
# stx-scripts/ <-- copy of build-docker-images/loci/docker/stx-scripts
# stx-wheels/ <-- copy of wheels tarball(s) specified on cmd line
#
echo "Patching ${WORKDIR}/loci/Dockerfile" >&2
# Make a copy of Dockerfile
\cp -f "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" || exit 1
# Replace "RUN .../install.sh" with our own commands
sed -i -r "${WORKDIR}/loci/Dockerfile.stx" -e "
\\%^\\s*(RUN|run)\\s+/opt/loci/scripts/install.sh\\s*\$% {
s/^(.*)$/#\\1/
r ${MY_SCRIPT_DIR}/loci/docker/Dockerfile.part
}
" || exit 1
if diff -q "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >/dev/null ; then
echo "${WORKDIR}/loci/Dockerfile: failed to patch Loci Dockerfile" >&2
exit 1
fi
# Copy stx-scripts to Loci dir
\rm -rf "${WORKDIR}/loci/stx-scripts" || exit 1
\cp -ar "${MY_SCRIPT_DIR}/loci/docker/stx-scripts" "${WORKDIR}/loci/" || exit 1
#diff -u "${WORKDIR}/loci/Dockerfile" "${WORKDIR}/loci/Dockerfile.stx" >&2
( cd "${WORKDIR}/loci" && git am $( \ls -1 $MY_SCRIPT_DIR/loci/patches/*.patch | sort ) ; )
# clear wheels dir
\rm -rf "${WORKDIR}/loci/stx-wheels" || exit 1
mkdir -p "${WORKDIR}/loci/stx-wheels" || exit 1
\rm -rf "${WORKDIR}/loci/stx-wheels/"* || exit 1
}
function download_loci_wheels {
@ -433,6 +397,10 @@ function build_image_loci {
SPICE_REF=$(source ${image_build_file} && echo ${SPICE_REF})
local DIST_REPOS
DIST_REPOS=$(source ${image_build_file} && echo ${DIST_REPOS})
local NON_UNIQUE_SYSTEM_ACCOUNT
NON_UNIQUE_SYSTEM_ACCOUNT=$(source ${image_build_file} && echo ${NON_UNIQUE_SYSTEM_ACCOUNT})
local UPDATE_SYSTEM_ACCOUNT
UPDATE_SYSTEM_ACCOUNT=$(source ${image_build_file} && echo ${UPDATE_SYSTEM_ACCOUNT})
echo "Building ${LABEL}"
@ -548,8 +516,13 @@ function build_image_loci {
BUILD_ARGS+=(--build-arg DIST_REPOS="${DIST_REPOS}")
fi
# Use patched docker file
BUILD_ARGS+=(--file "${WORKDIR}/loci/Dockerfile.stx")
if [ -n "${NON_UNIQUE_SYSTEM_ACCOUNT}" ]; then
BUILD_ARGS+=(--build-arg NON_UNIQUE_SYSTEM_ACCOUNT="${NON_UNIQUE_SYSTEM_ACCOUNT}")
fi
if [ -n "${UPDATE_SYSTEM_ACCOUNT}" ]; then
BUILD_ARGS+=(--build-arg UPDATE_SYSTEM_ACCOUNT="${UPDATE_SYSTEM_ACCOUNT}")
fi
# Disable build cache
if [[ "$USE_DOCKER_CACHE" != "yes" ]] ; then

View File

@ -1,5 +0,0 @@
ARG DIST_REPOS
COPY stx-scripts /opt/loci/stx-scripts
COPY stx-wheels /opt/loci/stx-wheels
RUN /opt/loci/stx-scripts/install.sh

View File

@ -1,6 +0,0 @@
#!/bin/bash
set -ex
rm -rf /opt/loci/stx-wheels/*

View File

@ -1,14 +0,0 @@
#!/bin/bash
set -ex
LOCI_DIR="/opt/loci"
# configure apt/yum repos
"$LOCI_DIR/stx-scripts/setup-package-repos.sh"
# run Loci installer
"$LOCI_DIR/scripts/install.sh" "$@"
# delete wheel tarball etc
"$LOCI_DIR/stx-scripts/cleanup.sh"

View File

@ -1,126 +0,0 @@
#!/bin/bash
set -ex
#
# This script enables or disables package repos specified
# by the DIST_REPOS environment variable, which must contain
# a space-separated list of repos (in CentOS) or list files
# (Debian) to enable or disable.
#
# In CentOS repo names refer to the names in square brackets
# in any repo files under /etc/yum.repos.d.
#
# In Debian repo names refer to individual files under
# /etc/apt/sources.list.d/$NAME.list.
#
# Repo names may be prefixed with
# a "+" (enable) or a "-" (disable). The leading "+" may be
# omitted.
#
# Additionally, the following keywords are treated specially:
#
# STX - enable or disable all StarlingX repos, ie
# the locally-built package repos, the mirror/download
# repo, and any repo's passed on the command-line
# to "build-stx-image.sh" script.
#
# OS - same as "base updates extras" in CentOS
# same as "debian" in Debian
#
#
# These keywords have the same meaning in all distros, while actual
# repo names are distro-specific.
#
# Any repos not included in $DIST_REPOS will remain unchanged (ie
# they will remain enabled or disabled as defined in the base image).
#
# If a repo doesn't match an existing repository, this script will
# fail.
#
# CentOS Example
# ==============
# DIST_REPOS="-base -updates"
# disable "base" and "updates" repos normally defined
# in /etc/yum.repos.d/CentOS-Base.repo
#
# DIST_REPOS="-STX +OS -updates"
# disable all local repos, enable core OS repos, except "updates"
#
# Debian Example
# ==============
# DIST_REPOS="debian"
# enable core OS repos (ie /etc/apt/sources.list.d/debian.list)
#
# DIST_REPOS="OS -STX"
# enable core OS repos (ie /etc/apt/sources.list.d/debian.list),
# disable STX repos (ie /etc/apt/sources.list.d/stx.list)
#
#
if [[ -n "$DIST_REPOS" ]] ; then
# basenames of files under /etc/apt/sources.list.d
declare -A DEBIAN_REPO_GROUPS=(
[OS]="debian"
[STX]="stx"
)
# yum repo IDs
declare -A CENTOS_REPO_GROUPS=(
[OS]="base updates extras"
[STX]="/etc/yum.repos.d/stx.repo" # ie, all repos defined in this file
)
distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release)
# enable or disable each repo
for base in $DIST_REPOS ; do
# starts with "-": disable this repo
if [[ "${base#-}" != "$base" ]] ; then
base="${base#-}"
enable=0
# starts with "+": enable this repo
elif [[ "${base#+}" != "$base" ]] ; then
base="${base#+}"
enable=1
# doesn't start with +/-: assume "+"
else
enable=1
fi
# enable or disable a repo
case ${distro} in
debian)
list_files="${DEBIAN_REPO_GROUPS[$base]:-$base}"
for list_file in $list_files ; do
if [[ $enable -eq 1 ]] ; then
cp -f /etc/apt/sources.list.d/${list_file}.list.disabled /etc/apt/sources.list.d/${list_file}.list
else
rm /etc/apt/sources.list.d/${list_file}.list
fi
done
;;
centos)
specs="${CENTOS_REPO_GROUPS[$base]:-$base}"
for spec in $specs ; do
# repo id begins with a "/" - assume its a full path to a .repo file
# and enable/disable all repos defined in that file
if [[ "${spec#/}" != "$spec" ]] ; then
repos=$(sed -r -n 's/^\s*[[]([^]]+)[]]\s*$/\1/gp' "$spec")
else
repos=$spec
fi
for repo in $repos ; do
if [[ $enable -eq 1 ]] ; then
yum-config-manager --enable "$repo"
else
yum-config-manager --disable "$repo"
fi
done
done
;;
*)
echo "error: unsupported OS \"$distro\"" >&2
exit 1
esac
done
fi

View File

@ -0,0 +1,215 @@
From 644ac10c8877444c540aac36111e59c65c47ce59 Mon Sep 17 00:00:00 2001
From: Davlet Panech <davlet.panech@windriver.com>
Date: Thu, 8 Sep 2022 21:04:55 +0000
Subject: [PATCH 1/2] starlingx: wheels from filesystem + pkg repos
- Allow WHEELS to be a local file path, rather than URL
- Dockerfile: new parameter DIST_REPOS that allows one to
enable/disable package repos when building
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
---
.gitignore | 1 +
Dockerfile | 6 +-
stx-scripts/cleanup.sh | 6 ++
stx-scripts/install.sh | 14 ++++
stx-scripts/setup-package-repos.sh | 126 +++++++++++++++++++++++++++++
stx-wheels/.keep | 0
6 files changed, 152 insertions(+), 1 deletion(-)
create mode 100644 .gitignore
create mode 100755 stx-scripts/cleanup.sh
create mode 100755 stx-scripts/install.sh
create mode 100755 stx-scripts/setup-package-repos.sh
create mode 100644 stx-wheels/.keep
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4b5032a
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+stx-wheels/[^.]*
diff --git a/Dockerfile b/Dockerfile
index 3a026a3..145d284 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -32,4 +32,8 @@ ARG SPICE_REF=${SPICE_REF:-spice-html5-0.1.6}
COPY scripts /opt/loci/scripts
ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/
-RUN /opt/loci/scripts/install.sh
+#RUN /opt/loci/scripts/install.sh
+ARG DIST_REPOS
+COPY stx-scripts /opt/loci/stx-scripts
+COPY stx-wheels /opt/loci/stx-wheels
+RUN /opt/loci/stx-scripts/install.sh
diff --git a/stx-scripts/cleanup.sh b/stx-scripts/cleanup.sh
new file mode 100755
index 0000000..6ac890f
--- /dev/null
+++ b/stx-scripts/cleanup.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+
+rm -rf /opt/loci/stx-wheels/*
+
diff --git a/stx-scripts/install.sh b/stx-scripts/install.sh
new file mode 100755
index 0000000..033bdb9
--- /dev/null
+++ b/stx-scripts/install.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -ex
+
+LOCI_DIR="/opt/loci"
+
+# configure apt/yum repos
+"$LOCI_DIR/stx-scripts/setup-package-repos.sh"
+
+# run Loci installer
+"$LOCI_DIR/scripts/install.sh" "$@"
+
+# delete wheel tarball etc
+"$LOCI_DIR/stx-scripts/cleanup.sh"
diff --git a/stx-scripts/setup-package-repos.sh b/stx-scripts/setup-package-repos.sh
new file mode 100755
index 0000000..dd43612
--- /dev/null
+++ b/stx-scripts/setup-package-repos.sh
@@ -0,0 +1,126 @@
+#!/bin/bash
+
+set -ex
+
+#
+# This script enables or disables package repos specified
+# by the DIST_REPOS environment variable, which must contain
+# a space-separated list of repos (in CentOS) or list files
+# (Debian) to enable or disable.
+#
+# In CentOS repo names refer to the names in square brackets
+# in any repo files under /etc/yum.repos.d.
+#
+# In Debian repo names refer to individual files under
+# /etc/apt/sources.list.d/$NAME.list.
+#
+# Repo names may be prefixed with
+# a "+" (enable) or a "-" (disable). The leading "+" may be
+# omitted.
+#
+# Additionally, the following keywords are treated specially:
+#
+# STX - enable or disable all StarlingX repos, ie
+# the locally-built package repos, the mirror/download
+# repo, and any repo's passed on the command-line
+# to "build-stx-image.sh" script.
+#
+# OS - same as "base updates extras" in CentOS
+# same as "debian" in Debian
+#
+#
+# These keywords have the same meaning in all distros, while actual
+# repo names are distro-specific.
+#
+# Any repos not included in $DIST_REPOS will remain unchanged (ie
+# they will remain enabled or disabled as defined in the base image).
+#
+# If a repo doesn't match an existing repository, this script will
+# fail.
+#
+# CentOS Example
+# ==============
+# DIST_REPOS="-base -updates"
+# disable "base" and "updates" repos normally defined
+# in /etc/yum.repos.d/CentOS-Base.repo
+#
+# DIST_REPOS="-STX +OS -updates"
+# disable all local repos, enable core OS repos, except "updates"
+#
+# Debian Example
+# ==============
+# DIST_REPOS="debian"
+# enable core OS repos (ie /etc/apt/sources.list.d/debian.list)
+#
+# DIST_REPOS="OS -STX"
+# enable core OS repos (ie /etc/apt/sources.list.d/debian.list),
+# disable STX repos (ie /etc/apt/sources.list.d/stx.list)
+#
+#
+
+if [[ -n "$DIST_REPOS" ]] ; then
+ # basenames of files under /etc/apt/sources.list.d
+ declare -A DEBIAN_REPO_GROUPS=(
+ [OS]="debian"
+ [STX]="stx"
+ )
+ # yum repo IDs
+ declare -A CENTOS_REPO_GROUPS=(
+ [OS]="base updates extras"
+ [STX]="/etc/yum.repos.d/stx.repo" # ie, all repos defined in this file
+ )
+
+ distro=$(awk -F= '/^ID=/ {gsub(/\"/, "", $2); print $2}' /etc/*release)
+ # enable or disable each repo
+ for base in $DIST_REPOS ; do
+ # starts with "-": disable this repo
+ if [[ "${base#-}" != "$base" ]] ; then
+ base="${base#-}"
+ enable=0
+ # starts with "+": enable this repo
+ elif [[ "${base#+}" != "$base" ]] ; then
+ base="${base#+}"
+ enable=1
+ # doesn't start with +/-: assume "+"
+ else
+ enable=1
+ fi
+
+ # enable or disable a repo
+ case ${distro} in
+ debian)
+ list_files="${DEBIAN_REPO_GROUPS[$base]:-$base}"
+ for list_file in $list_files ; do
+ if [[ $enable -eq 1 ]] ; then
+ cp -f /etc/apt/sources.list.d/${list_file}.list.disabled /etc/apt/sources.list.d/${list_file}.list
+ else
+ rm /etc/apt/sources.list.d/${list_file}.list
+ fi
+ done
+ ;;
+ centos)
+ specs="${CENTOS_REPO_GROUPS[$base]:-$base}"
+ for spec in $specs ; do
+ # repo id begins with a "/" - assume its a full path to a .repo file
+ # and enable/disable all repos defined in that file
+ if [[ "${spec#/}" != "$spec" ]] ; then
+ repos=$(sed -r -n 's/^\s*[[]([^]]+)[]]\s*$/\1/gp' "$spec")
+ else
+ repos=$spec
+ fi
+ for repo in $repos ; do
+ if [[ $enable -eq 1 ]] ; then
+ yum-config-manager --enable "$repo"
+ else
+ yum-config-manager --disable "$repo"
+ fi
+ done
+ done
+ ;;
+ *)
+ echo "error: unsupported OS \"$distro\"" >&2
+ exit 1
+ esac
+ done
+fi
+
diff --git a/stx-wheels/.keep b/stx-wheels/.keep
new file mode 100644
index 0000000..e69de29
--
2.30.2

View File

@ -0,0 +1,122 @@
From e2f044c29b7c5d08113ee633abf7c3b9680d583d Mon Sep 17 00:00:00 2001
From: Davlet Panech <davlet.panech@windriver.com>
Date: Thu, 8 Sep 2022 21:13:35 +0000
Subject: [PATCH 2/2] Don't fail if user/group exists in base image
New parameter NON_UNIQUE_SYSTEM_ACCOUNT: allow creation of user/group
with non-unique IDs
New parameter UPDATE_SYSTEM_ACCOUNT: update UID/GID/HOME if account
already exists and doesn't match what we expect.
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
---
Dockerfile | 2 ++
scripts/create_user.sh | 77 ++++++++++++++++++++++++++++++++++++++++--
2 files changed, 77 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 145d284..6567c90 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,6 +20,8 @@ ARG EXTRA_PYDEP=""
ARG REGISTRY_PROTOCOL="detect"
ARG REGISTRY_INSECURE="False"
ARG KEEP_ALL_WHEELS="False"
+ARG UPDATE_SYSTEM_ACCOUNT="no"
+ARG NON_UNIQUE_SYSTEM_ACCOUNT="no"
ARG UID=42424
ARG GID=42424
diff --git a/scripts/create_user.sh b/scripts/create_user.sh
index 417875d..fb59bd8 100755
--- a/scripts/create_user.sh
+++ b/scripts/create_user.sh
@@ -1,9 +1,82 @@
#!/bin/bash
+#
+# UPDATE_SYSTEM_ACCOUNT: yes/no, default=no
+# if "yes":
+# if user/group exist, change their UID, GID & home dir
+# else:
+# if user/group exist, and their UID/UID/home dir are not what
+# we expect, then fail
+#
+# NON_UNIQUE_SYSTEM_ACCOUNT: yes/no, default=no
+# if yes: allow non-unique UID/GUID
+
set -ex
-groupadd -g ${GID} ${PROJECT}
-useradd -u ${UID} -g ${PROJECT} -M -d /var/lib/${PROJECT} -s /usr/sbin/nologin -c "${PROJECT} user" ${PROJECT}
+if [[ "$NON_UNIQUE_SYSTEM_ACCOUNT" == "yes" ]] ; then
+ non_unique="-o"
+fi
+
+# Group exists?
+if grent="$(getent group ${PROJECT})" ; then
+ # make sure GID matches
+ gid=$(echo "$grent" | awk -v FS=: '{print $3}')
+ if [[ $gid != $GID ]] ; then
+ if [[ "$UPDATE_SYSTEM_ACCOUNT" != "yes" ]] ; then
+ echo "Group ${PROJECT} already exists and has an unexpected GID $gid (expecting: $GID)" >&2
+ exit 1
+ fi
+ echo "## group ${PROJECT}: changing gid $gid -> $GID" >&2
+ groupmod $non_unique -g "$GID" "${PROJECT}"
+ fi
+# no group: create it
+else
+ echo "## group ${PROJECT}: creating gid=$gid" >&2
+ groupadd $non_unique -g ${GID} ${PROJECT}
+fi
+
+# User exists?
+if pwent="$(getent passwd "${PROJECT}")" ; then
+ # make sure GID, UID & home dir match
+ uid=$(echo "$pwent" | awk -v FS=: '{ print $3 }')
+ gid=$(echo "$pwent" | awk -v FS=: '{ print $4 }')
+ homedir=$(echo "$pwent" | awk -v FS=: '{ print $6 }')
+ # check UID
+ if [[ "$uid" != "$UID" ]] ; then
+ if [[ "$UPDATE_SYSTEM_ACCOUNT" != "yes" ]] ; then
+ echo "User ${PROJECT} already exists and has an unexpected UID $uid (expecting: $UID)" >&2
+ exit 1
+ fi
+ echo "## user ${PROJECT}: changing uid $uid -> $UID" >&2
+ usermod $non_unique -u "$UID" "${PROJECT}"
+ fi
+ # check GID
+ if [[ "$gid" != "$GID" ]] ; then
+ if [[ "$UPDATE_SYSTEM_ACCOUNT" != "yes" ]] ; then
+ echo "User ${PROJECT} already exists and has an unexpected GID $gid (expecting: $GID)" >&2
+ exit 1
+ fi
+ echo "## user ${PROJECT}: changing gid $gid -> $GID" >&2
+ usermod -g "$GID" "${PROJECT}"
+ fi
+ # check home dir
+ # see https://www.gnu.org/software/coreutils/manual/html_node/realpath-invocation.html#realpath-invocation
+ canon_homedir="$(realpath --canonicalize-missing --no-symlinks "$homedir")"
+ canon_ref_homedir="/var/lib/${PROJECT}"
+ if [[ "$canon_homedir" != "$canon_ref_homedir" ]] ; then
+ if [[ "$UPDATE_SYSTEM_ACCOUNT" != "yes" ]] ; then
+ echo "User ${PROJECT} already exists and has an unexpected home directory $homedir (expecting: /var/lib/${PROJECT}" >&2
+ exit 1
+ fi
+ echo "## user ${PROJECT}: changing home $homedir -> /var/lib/${PROJECT}" >&2
+ usermod -d "/var/lib/${PROJECT}" "${PROJECT}"
+ fi
+# no user: create it
+else
+ echo "## user ${PROJECT}: creating uid=$UID gid=$GID home=/var/lib/${PROJECT}" >&2
+ useradd $non_unique -u "${UID}" -g "${PROJECT}" -M -d "/var/lib/${PROJECT}" -s /usr/sbin/nologin -c "${PROJECT} user" "${PROJECT}"
+fi
+# create any missing dirs
mkdir -p /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
chown ${PROJECT}:${PROJECT} /etc/${PROJECT} /var/log/${PROJECT} /var/lib/${PROJECT} /var/cache/${PROJECT}
--
2.30.2