From f1d7e902e39264a0dd9584d7da11e7e62339d0a6 Mon Sep 17 00:00:00 2001 From: Matthew Thode Date: Fri, 26 Jul 2019 17:18:40 -0500 Subject: [PATCH] support alternate portage directories The 17.1 profile changed the defaults used in portage for where we store our repo, distfiles and binpkgs. Some portage related variables need to be set deterministically. 17.1 is no enabled for Systemd's profile. Change-Id: Ib55f6875c5cb461c3c530b51d7420ce3dc8da360 --- .../elements/gentoo/bin/install-packages | 25 ++++++++++++++++--- .../elements/gentoo/cleanup.d/05-unmount | 8 +++--- .../environment.d/00-gentoo-envars.bash | 12 ++++++--- .../elements/gentoo/finalise.d/99-cleanup | 5 +++- .../gentoo/pre-finalise.d/01-gentoo-cache | 11 ++++---- .../pre-install.d/02-gentoo-00-set-profile | 10 ++++++++ .../02-gentoo-03-enable-overlays | 2 +- .../02-gentoo-04-install-desired-python | 11 +++----- .../elements/gentoo/root.d/10-gentoo-image | 18 ++++++------- .../elements/gentoo/root.d/50-gentoo-cache | 11 ++++---- 10 files changed, 75 insertions(+), 38 deletions(-) create mode 100755 diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-00-set-profile diff --git a/diskimage_builder/elements/gentoo/bin/install-packages b/diskimage_builder/elements/gentoo/bin/install-packages index 7b25e3065..a57eaa75d 100755 --- a/diskimage_builder/elements/gentoo/bin/install-packages +++ b/diskimage_builder/elements/gentoo/bin/install-packages @@ -20,6 +20,25 @@ fi set -eu set -o pipefail +# env is not sourced with user specified package inclusions +# set the default bash array if GENTOO_EMERGE_ENV is not defined as an array +if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then + declare -a GENTOO_EMERGE_ENV + GENTOO_EMERGE_ENV+=("USE=\"-build\"") + GENTOO_EMERGE_ENV+=("FEATURES=\"binpkg-multi-instance buildpkg parallel-fetch parallel-install\"") + GENTOO_EMERGE_ENV+=("PKGDIR=\"/tmp/portage-pkgdir\"") + GENTOO_EMERGE_ENV+=("DISTDIR=\"/tmp/portage-distdir\"") + GENTOO_EMERGE_ENV+=("PORTDIR=\"/tmp/portage-portdir\"") + export GENTOO_EMERGE_ENV + #export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"') +fi +# itterate over the array, exporting each 'line' +for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do + eval export "${GENTOO_EMERGE_ENV[i]}" +done +# set default opts +GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2 --autounmask=n"} + ACTION='install' SCRIPTNAME="$(basename $0)" @@ -55,7 +74,7 @@ eval set -- "${TEMP}" while true; do case "${1}" in -u ) - if [[ ! -f /usr/portage/profiles ]]; then + if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages --update --deep --newuse --complete-graph=y @world @@ -100,12 +119,12 @@ if [[ -z "${PKGS}" ]]; then echo "Not running install-packages ${ACTION} with empty packages list" else if [[ "${ACTION}" == 'install' ]]; then - if [[ ! -f /usr/portage/profiles ]]; then + if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages -q --jobs=2 --changed-use "${PKGS}" elif [[ "${ACTION}" == 'remove' ]]; then - if [[ ! -f /usr/portage/profiles ]]; then + if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi install_gentoo_packages -C "${PKGS}" diff --git a/diskimage_builder/elements/gentoo/cleanup.d/05-unmount b/diskimage_builder/elements/gentoo/cleanup.d/05-unmount index 9b720380a..47d4c78dc 100755 --- a/diskimage_builder/elements/gentoo/cleanup.d/05-unmount +++ b/diskimage_builder/elements/gentoo/cleanup.d/05-unmount @@ -8,15 +8,15 @@ set -o pipefail [[ -n "${TMP_MOUNT_PATH}" ]] -sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/distfiles -sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage/packages -sudo umount -f "${TMP_MOUNT_PATH}"/usr/portage +sudo umount -f "${TMP_MOUNT_PATH}${DISTDIR}" +sudo umount -f "${TMP_MOUNT_PATH}${PKGDIR}" +sudo umount -f "${TMP_MOUNT_PATH}${PORTDIR}" sudo rm -Rf "${TMP_MOUNT_PATH}"/root/.ccache/* "${TMP_MOUNT_PATH}"/usr/src/* "${TMP_MOUNT_PATH}"/var/cache/edb/dep/* "${TMP_MOUNT_PATH}"/var/cache/genkernel/* "${TMP_MOUNT_PATH}"/var/empty/* "${TMP_MOUNT_PATH}"/var/state/* "${TMP_MOUNT_PATH}"/var/tmp/* sudo rm -Rf "${TMP_MOUNT_PATH}"/etc/*- "${TMP_MOUNT_PATH}"/etc/*.old "${TMP_MOUNT_PATH}"/etc/ssh/ssh_host_* "${TMP_MOUNT_PATH}"/root/.*history "${TMP_MOUNT_PATH}"/root/.lesshst "${TMP_MOUNT_PATH}"/root/.ssh/known_hosts "${TMP_MOUNT_PATH}"/root/.viminfo "${TMP_MOUNT_PATH}"/usr/share/genkernel "${TMP_MOUNT_PATH}"/usr/lib64/python*/site-packages/gentoolkit/test/eclean/testdistfiles.tar.gz if [[ "${GENTOO_PORTAGE_CLEANUP}" != "False" ]]; then # remove portage files - sudo rm -Rf "${TMP_MOUNT_PATH}"/usr/portage/* "${TMP_MOUNT_PATH}"/usr/portage/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles + sudo rm -Rf "${TMP_MOUNT_PATH}${PORTDIR}"/* "${TMP_MOUNT_PATH}${PORTDIR}"/.git "${TMP_MOUNT_PATH}"/var/cache/portage/distfiles "${TMP_MOUNT_PATH}"/usr/portage/* fi # shrink a bit diff --git a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash index fc56255e8..74a308cee 100644 --- a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash +++ b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash @@ -1,15 +1,21 @@ export DIB_RELEASE=gentoo export DISTRO_NAME=gentoo -export GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.0'} +export GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'} export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-'python2_7 python3_6'} export GENTOO_PYTHON_ACTIVE_VERSION=${GENTOO_PYTHON_ACTIVE_VERSION:-'python3.6'} export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''} -export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2"} +export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2 --autounmask=n"} # set the default bash array if GENTOO_EMERGE_ENV is not defined as an array if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then - export GENTOO_EMERGE_ENV=('USE="-build"' 'FEATURES="binpkg-multi-instance buildpkg parallel-fetch parallel-install"' 'EMERGE_DEFAULT_OPTS="--autounmask=n"') + declare -a GENTOO_EMERGE_ENV + GENTOO_EMERGE_ENV+=("USE=\"-build\"") + GENTOO_EMERGE_ENV+=("FEATURES=\"binpkg-multi-instance buildpkg parallel-fetch parallel-install\"") + GENTOO_EMERGE_ENV+=("PKGDIR=\"/tmp/portage-pkgdir\"") + GENTOO_EMERGE_ENV+=("DISTDIR=\"/tmp/portage-distdir\"") + GENTOO_EMERGE_ENV+=("PORTDIR=\"/tmp/portage-portdir\"") + export GENTOO_EMERGE_ENV fi # itterate over the array, exporting each 'line' for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do diff --git a/diskimage_builder/elements/gentoo/finalise.d/99-cleanup b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup index 0f9217df4..ba42a0b88 100755 --- a/diskimage_builder/elements/gentoo/finalise.d/99-cleanup +++ b/diskimage_builder/elements/gentoo/finalise.d/99-cleanup @@ -15,7 +15,7 @@ else fi # sync if needed -[[ ! -e /usr/portage/profiles ]] && emerge-webrsync +[[ ! -e ${PORTDIR}/profiles ]] && emerge-webrsync # make world consistent emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --complete-graph=y --update --newuse --deep @world @@ -47,3 +47,6 @@ unfix_shm # remove within the chroot since /var/run is a symlink rm -Rf /var/run/* /run/* + +# after updating, set profile to where PORTDIR actually is (varies by profile) +ln -sf ../.."$(env -u PORTDIR portageq get_repo_path / gentoo)/profiles/${GENTOO_PROFILE}" /etc/portage/make.profile diff --git a/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache b/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache index 32887405c..8982bbf4c 100755 --- a/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache +++ b/diskimage_builder/elements/gentoo/pre-finalise.d/01-gentoo-cache @@ -10,8 +10,9 @@ GENTOO_PORTAGE_DISTFILES_DIR="${DIB_IMAGE_CACHE}/gentoo/distfiles" GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE}" GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage" -sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage -mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles -sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles -mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages -sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages +mkdir -p "${TMP_MOUNT_PATH}${PORTDIR}" +sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}${PORTDIR}" +mkdir -p "${TMP_MOUNT_PATH}${DISTDIR}" +sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}${DISTDIR}" +mkdir -p "${TMP_MOUNT_PATH}${PKGDIR}" +sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}${PKGDIR}" diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-00-set-profile b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-00-set-profile new file mode 100755 index 000000000..015e53731 --- /dev/null +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-00-set-profile @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# set profile symlink to the hardcoded PORDIR +ln -sf ../.."${PORTDIR}/profiles/${GENTOO_PROFILE}" /etc/portage/make.profile diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays index 347bfe476..7f843c84e 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-03-enable-overlays @@ -15,7 +15,7 @@ if [[ ${GENTOO_OVERLAYS} != '' ]]; then fix_shm fi - if [[ ! -f /usr/portage/profiles ]]; then + if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python index 5c51f5594..fde74d7c0 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-04-install-desired-python @@ -14,9 +14,7 @@ else fix_shm fi -if [[ ! -f /usr/portage/profiles ]]; then - emerge-webrsync -q -fi +[[ ! -f ${PORTDIR}/profiles ]] && emerge-webrsync -q echo "PYTHON_TARGETS=\"${GENTOO_PYTHON_TARGETS}\"" >> /etc/portage/make.conf [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_6"* ]] && [[ ! -e /usr/lib64/libpython3.6m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.6 @@ -24,12 +22,11 @@ echo "PYTHON_TARGETS=\"${GENTOO_PYTHON_TARGETS}\"" >> /etc/portage/make.conf [[ "${GENTOO_PYTHON_TARGETS}" == *"python3_4"* ]] && [[ ! -e /usr/lib64/libpython3.4m.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:3.4 [[ "${GENTOO_PYTHON_TARGETS}" == *"python2_7"* ]] && [[ ! -e /usr/lib64/libpython2.7.so ]] && emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot dev-lang/python:2.7 # disable python in git if we have to, it only supports python-2.7 -if [[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]]; then - echo 'dev-vcs/git -python' >> /etc/portage/package.use/git -fi +[[ "${GENTOO_PYTHON_TARGETS}" != *"python2_7"* ]] && echo 'dev-vcs/git -python' >> /etc/portage/package.use/git + # make sure we have the new python for portage bevore we possibly remove python 2.7 # this is done one by one with nodeps because it can sometimes have dependency loops -PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/pyparsing dev-python/appdirs dev-python/setuptools dev-python/certifi" +PYTHON_PACKAGES="dev-python/six dev-python/packaging dev-python/appdirs dev-python/setuptools dev-python/pyparsing dev-python/certifi" for PACKAGE in ${PYTHON_PACKAGES}; do emerge ${GENTOO_EMERGE_DEFAULT_OPTS} --oneshot --update --newuse --deep --nodeps "${PACKAGE}" done diff --git a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image index a278695df..482c28143 100755 --- a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image +++ b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image @@ -38,17 +38,17 @@ fi # default/linux/amd64/13.0/no-multilib # hardened/linux/amd64 # hardened/linux/amd64/no-multilib -GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.0'} -if [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0" ]]; then +GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} +if [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1" ]]; then FILENAME_BASE='gentoo-stage4' SIGNED_SOURCE_SUFFIX='minimal' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/no-multilib" ]]; then +elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib" ]]; then FILENAME_BASE='gentoo-stage4-nomultilib' SIGNED_SOURCE_SUFFIX='minimal-nomultilib' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/hardened" ]]; then +elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/hardened" ]]; then FILENAME_BASE='gentoo-stage4-hardened' SIGNED_SOURCE_SUFFIX='hardened+minimal' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/no-multilib/hardened" ]]; then +elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib/hardened" ]]; then FILENAME_BASE='gentoo-stage4-hardened-nomultilib' SIGNED_SOURCE_SUFFIX='hardened+minimal-nomultilib' elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/systemd" ]]; then @@ -56,10 +56,10 @@ elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/systemd" ]]; then SIGNED_SOURCE_SUFFIX='systemd' else echo 'invalid profile, please select from the following profiles' - echo 'default/linux/amd64/17.0' - echo 'default/linux/amd64/17.0/no-multilib' - echo 'default/linux/amd64/17.0/hardened' - echo 'default/linux/amd64/17.0/no-multilib/hardened' + echo 'default/linux/amd64/17.1' + echo 'default/linux/amd64/17.1/no-multilib' + echo 'default/linux/amd64/17.1/hardened' + echo 'default/linux/amd64/17.1/no-multilib/hardened' echo 'default/linux/amd64/17.0/systemd' exit 1 fi diff --git a/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache b/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache index 84143016e..4de11dff8 100755 --- a/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache +++ b/diskimage_builder/elements/gentoo/root.d/50-gentoo-cache @@ -13,8 +13,9 @@ GENTOO_PORTAGE_PACKAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/packages/${GENTOO_PROFILE} GENTOO_PORTAGE_DIR="${DIB_IMAGE_CACHE}/gentoo/portage" mkdir -p "${GENTOO_PORTAGE_DIR}" "${GENTOO_PORTAGE_PACKAGE_DIR}" "${GENTOO_PORTAGE_DISTFILES_DIR}" -sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage -mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/distfiles -sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/distfiles -mkdir -p "${TMP_MOUNT_PATH}"/usr/portage/packages -sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}"/usr/portage/packages +mkdir -p "${TMP_MOUNT_PATH}${PORTDIR}" +sudo mount --bind "${GENTOO_PORTAGE_DIR}" "${TMP_MOUNT_PATH}${PORTDIR}" +mkdir -p "${TMP_MOUNT_PATH}${DISTDIR}" +sudo mount --bind "${GENTOO_PORTAGE_DISTFILES_DIR}" "${TMP_MOUNT_PATH}${DISTDIR}" +mkdir -p "${TMP_MOUNT_PATH}${PKGDIR}" +sudo mount --bind "${GENTOO_PORTAGE_PACKAGE_DIR}" "${TMP_MOUNT_PATH}${PKGDIR}"