diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index d9f8da9fc..2b7dd0f36 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -266,8 +266,6 @@ nodepool_diskimage: base_element: gentoo release: '' - env-vars: - GENTOO_PROFILE: 'default/linux/amd64/17.1/systemd/merged-usr' - job: name: dib-nodepool-functional-openstack-debian-stretch-src diff --git a/diskimage_builder/elements/devuser/install.d/50-devuser b/diskimage_builder/elements/devuser/install.d/50-devuser index 5e704ef22..8819b34cc 100755 --- a/diskimage_builder/elements/devuser/install.d/50-devuser +++ b/diskimage_builder/elements/devuser/install.d/50-devuser @@ -19,6 +19,7 @@ fi set -x if [ -n "${DIB_DEV_USER_PWDLESS_SUDO}" ]; then + mkdir -p /etc/sudoers.d/ cat > /etc/sudoers.d/${DIB_DEV_USER_USERNAME} << EOF ${DIB_DEV_USER_USERNAME} ALL=(ALL) NOPASSWD:ALL EOF diff --git a/diskimage_builder/elements/gentoo/README.rst b/diskimage_builder/elements/gentoo/README.rst index 1719076e2..5a81fec58 100644 --- a/diskimage_builder/elements/gentoo/README.rst +++ b/diskimage_builder/elements/gentoo/README.rst @@ -23,25 +23,29 @@ Notes: * In order to run the vm element you will need to make sure `sys-block/parted` is installed on the host. -* Other profiles can be used by exporting GENTOO_PROFILE with a valid profile. - A list of valid profiles follows: +* The default profile is ``default/linux/amd64/23.0``. - default/linux/amd64/17.1 - default/linux/amd64/17.1/no-multilib - default/linux/amd64/17.1/hardened - default/linux/amd64/17.1/no-multilib/hardened - default/linux/amd64/17.1/systemd - default/linux/arm64/17.0 - default/linux/arm64/17.0/systemd +* Any ``amd64`` or ``arm64`` profile with a stage tarball published by gentoo + in the ``autobuilds`` directory for that arch are supported. Warning: + the GENTOO_PROFILE environment variable will take precedence over the ARCH + environment variable. * You can set the `GENTOO_PORTAGE_CLEANUP` environment variable to False to disable the clean up of portage repositories (including overlays). This will make the image bigger if caching is also disabled. +* In many cases, the resulting image will not have a valid profile set. If + you need to interactively use portage in a machine created with DIB, you + will need to run `eselect profile set some/valid/profile` before interacting + with portage. + * Gentoo supports many different versions of python, in order to select one you may use the `GENTOO_PYTHON_TARGETS` environment variable to select the versions of python you want on your image. The format of this variable - is a string as follows `"python2_7 python3_6"`. + is a string as follows `"python3_10 python3_11"`. This variable only impacts + the python versions used for distribution-installed python packages; see + https://wiki.gentoo.org/wiki/Project:Python/PYTHON_TARGETS for more + information. * You can enable overlays using the `GENTOO_OVERLAYS` variable. In it you should put a space separated list of overlays. The overlays must be in the diff --git a/diskimage_builder/elements/gentoo/bin/install-packages b/diskimage_builder/elements/gentoo/bin/install-packages index 43e0c44ff..d22cc5281 100755 --- a/diskimage_builder/elements/gentoo/bin/install-packages +++ b/diskimage_builder/elements/gentoo/bin/install-packages @@ -87,7 +87,7 @@ while true; do install_gentoo_packages --usepkg=n @preserved-rebuild etc-update --automode -5 eselect news read new - exit 0; + exit 0 ;; -e ) ACTION='remove' @@ -127,24 +127,33 @@ else if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi - install_gentoo_packages --changed-use "${PKGS}" + # --noreplace prevents us from rebuilding a package already installed + # --changed-use means that package will be rebuilt *if* USE flags for + # it (configuration) has changed + install_gentoo_packages --noreplace --changed-use "${PKGS}" elif [[ "${ACTION}" == 'remove' ]]; then if [[ ! -f ${PORTDIR}/profiles ]]; then emerge-webrsync -q fi - # remove packages from uninstall list that are not already installed - # this can result in false positives if not presented with full category/package names - CLEANED_PKGS=() + # A good practice for removing packages in gentoo is to deselect them, + # removing them from "world" set -- the equivalent of "unmark" in dnf. + # This tells portage we no longer care if the package is installed, + # and it can be removed if depedancies allow. + # This means a removal is two steps: + # - emerge --deselect $pkg + # - emerge --depclean + # + # The depclean step removes all packages that are not in the "world" + # set and are not in the dependency graph for any packages in "world" + # set. + # + # Other methods of removal may work; but this method sets us up to + # calculate the dependency graph exactly once and prevents portage + # from erroring if any of the packages were not already selected. for PKG in ${PKGS}; do - # the '^' and '$' in the search query are important so we don't get matched to - # packages that include our package name as part of their package name - if ! emerge --search "^${PKG}$" | grep -q 'Not Installed' ; then - CLEANED_PKGS+=("${PKG}") - fi + install_gentoo_packages --deselect $PKG done - if [ ${#CLEANED_PKGS[@]} -ne 0 ]; then - install_gentoo_packages -C "${CLEANED_PKGS[@]}" - fi + install_gentoo_packages --depclean else echo 'something went really wrong, install action is not install or remove' fi diff --git a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash index 8cc9c7e54..92941eb60 100644 --- a/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash +++ b/diskimage_builder/elements/gentoo/environment.d/00-gentoo-envars.bash @@ -1,12 +1,18 @@ export DIB_RELEASE=gentoo export DISTRO_NAME=gentoo export EFI_BOOT_DIR="EFI/gentoo" -export GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} + export GENTOO_PORTAGE_CLEANUP=${GENTOO_PORTAGE_CLEANUP:-'True'} export GENTOO_PYTHON_TARGETS=${GENTOO_PYTHON_TARGETS:-''} export GENTOO_OVERLAYS=${GENTOO_OVERLAYS:-''} export GENTOO_EMERGE_DEFAULT_OPTS=${GENTOO_EMERGE_DEFAULT_OPTS:-"--binpkg-respect-use --rebuilt-binaries=y --usepkg=y --with-bdeps=y --binpkg-changed-deps=y --quiet --jobs=2 --autounmask=n"} +# NOTE(JayF): This defines the base gentoo profile version supported +# in DIB. As gentoo is a rolling release distro, the older profiles +# are unsupported. +export GENTOO_BASE_PROFILE="default/linux/${ARCH}/23.0" +export GENTOO_PROFILE=${GENTOO_PROFILE:-$GENTOO_BASE_PROFILE} + # set the default bash array if GENTOO_EMERGE_ENV is not defined as an array if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then declare -a GENTOO_EMERGE_ENV @@ -17,7 +23,7 @@ if ! declare -p GENTOO_EMERGE_ENV 2> /dev/null | grep -q '^declare \-a'; then GENTOO_EMERGE_ENV+=("PORTDIR=\"/tmp/portage-portdir\"") export GENTOO_EMERGE_ENV fi -# itterate over the array, exporting each 'line' +# iterate over the array, exporting each 'line' for (( i=0; i<${#GENTOO_EMERGE_ENV[@]}; i++ )); do eval export "${GENTOO_EMERGE_ENV[i]}" done diff --git a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags index 7cf8ebba1..6386afa1a 100755 --- a/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags +++ b/diskimage_builder/elements/gentoo/pre-install.d/02-gentoo-02-flags @@ -20,21 +20,13 @@ mkdir -p /etc/portage/package.use echo 'dev-python/pip vanilla' >> /etc/portage/package.use/pip # needed to create disk images echo 'sys-fs/lvm2 lvm -thin' >> /etc/portage/package.use/grub -echo 'sys-kernel/installkernel dracut' >> /etc/portage/package.use/kernel +echo 'sys-kernel/installkernel grub dracut' >> /etc/portage/package.use/kernel echo 'sys-boot/grub device-mapper' >> /etc/portage/package.use/grub echo 'sys-boot/grub grub_platforms_efi-64' >> /etc/portage/package.use/grub # always enable efi-64 if [[ 'amd64' == "${ARCH}" ]]; then echo 'sys-boot/grub grub_platforms_pc' >> /etc/portage/package.use/grub # bios support for bios systems fi -# needed to install static kernel -echo "sys-kernel/gentoo-kernel-bin ~${ARCH}" >> /etc/portage/package.accept_keywords/kernel -echo "virtual/dist-kernel ~${ARCH}" >> /etc/portage/package.accept_keywords/kernel - -# needed for gcc-10 support -echo "~sys-block/open-iscsi-2.1.4 ~${ARCH}" >> /etc/portage/package.accept_keywords/open-iscsi -echo "~sys-block/open-isns-0.101 ~${ARCH}" >> /etc/portage/package.accept_keywords/open-iscsi - # musl only valid for amd64 for now if [[ "${GENTOO_PROFILE}" == *"musl"* ]]; then echo "sys-libs/pam cracklib" >> /etc/portage/package.use/musl diff --git a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image index 456e44c69..4017e95ec 100755 --- a/diskimage_builder/elements/gentoo/root.d/10-gentoo-image +++ b/diskimage_builder/elements/gentoo/root.d/10-gentoo-image @@ -24,56 +24,30 @@ set -o pipefail [ -n "${ARCH}" ] [ -n "${TARGET_ROOT}" ] -if [[ 'amd64' != "${ARCH}" ]] && [[ 'arm64' != "${ARCH}" ]]; then - echo "Only amd64 or arm64 images are currently available but ARCH is set to ${ARCH}." +P_SUFFIX="${GENTOO_PROFILE#$GENTOO_BASE_PROFILE}" +F_SUFFIX="${P_SUFFIX//\//\-}" +if [[ ${F_SUFFIX} != *"-systemd" ]]; then + # NOTE(JayF): OpenRC is implied, and appended to the filename, unless systemd is specified. + F_SUFFIX="${F_SUFFIX}-openrc" +fi + +DIB_CLOUD_SOURCE=${DIB_CLOUD_SOURCE:-"https://distfiles.gentoo.org/releases/${ARCH}/autobuilds/latest-stage3-${ARCH}${F_SUFFIX}.txt"} +echo "Fetching available stages from ${DIB_CLOUD_SOURCE} for profile ${GENTOO_PROFILE}" + +STAGE_LIST=$(curl "${DIB_CLOUD_SOURCE}" -s -f || true) +if [[ -z ${STAGE_LIST} ]]; then + echo "Unable to find a stage list for ${GENTOO_PROFILE} at ${DIB_CLOUD_SOURCE}." + echo "This element only currently supports profiles included in the periodic" + echo "Gentoo autobuilds." exit 1 fi -GENTOO_PROFILE=${GENTOO_PROFILE:-'default/linux/amd64/17.1'} -if [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1" ]]; then - FILENAME_BASE='amd64_gentoo-stage3' - SIGNED_SOURCE_SUFFIX='-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-nomultilib' - SIGNED_SOURCE_SUFFIX='-nomultilib-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened' - SIGNED_SOURCE_SUFFIX='-hardened-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/no-multilib/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened-nomultilib' - SIGNED_SOURCE_SUFFIX='-hardened-nomultilib-openrc' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.0/musl/hardened" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-hardened-musl' - SIGNED_SOURCE_SUFFIX='-musl-hardened' -elif [[ "${GENTOO_PROFILE}" == "default/linux/amd64/17.1/systemd/merged-usr" ]]; then - FILENAME_BASE='amd64_gentoo-stage3-systemd-mergedusr' - SIGNED_SOURCE_SUFFIX='-systemd-mergedusr' -elif [[ "${GENTOO_PROFILE}" == "default/linux/arm64/17.0" ]]; then - FILENAME_BASE='arm64_gentoo-stage3' - SIGNED_SOURCE_SUFFIX='' -elif [[ "${GENTOO_PROFILE}" == "default/linux/arm64/17.0/systemd/merged-usr" ]]; then - FILENAME_BASE='arm64_gentoo-stage3-systemd-mergedusr' - SIGNED_SOURCE_SUFFIX='-systemd-mergedusr' -else - echo 'invalid profile, please select from the following profiles' - echo 'default/linux/amd64/17.1' - echo 'default/linux/amd64/17.1/no-multilib' - echo 'default/linux/amd64/17.1/hardened' - echo 'default/linux/amd64/17.1/no-multilib/hardened' - echo 'default/linux/amd64/17.1/systemd/merged-usr' - echo 'default/linux/arm64/17.0' - echo 'default/linux/arm64/17.0/systemd/merged-usr' - exit 1 -fi +UPSTREAM_FILENAME=$(echo "${STAGE_LIST}" | grep -B1 'BEGIN PGP SIGNATURE' | head -n1 | cut -d\ -f1) -if [[ "${GENTOO_PROFILE}" == *'amd64'* ]]; then - ARCH_PATH='amd64' -elif [[ "${GENTOO_PROFILE}" == *'arm64'* ]]; then - ARCH_PATH='arm64' -fi -DIB_CLOUD_SOURCE=${DIB_CLOUD_SOURCE:-"http://distfiles.gentoo.org/releases/${ARCH_PATH}/autobuilds/latest-stage3-${ARCH_PATH}${SIGNED_SOURCE_SUFFIX}.txt"} -BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"http://distfiles.gentoo.org/releases/${ARCH_PATH}/autobuilds/$(curl "${DIB_CLOUD_SOURCE}" -s -f | grep -B1 'BEGIN PGP SIGNATURE' | head -n 1 | cut -d\ -f 1)"} +echo "Chose ${UPSTREAM_FILENAME} as candidate stage tarball" +BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"https://distfiles.gentoo.org/releases/${ARCH}/autobuilds/${UPSTREAM_FILENAME}"} BASE_IMAGE_FILE_SUFFIX=${BASE_IMAGE_FILE_SUFFIX:-"$(basename "${BASE_IMAGE_FILE}" | cut -d. -f 2,3)"} +FILENAME_BASE="gentoo-${GENTOO_PROFILE//\//\-}.${BASE_IMAGE_FILE_SUFFIX}" SIGNATURE_FILE="${SIGNATURE_FILE:-${BASE_IMAGE_FILE}.asc}" CACHED_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.${BASE_IMAGE_FILE_SUFFIX}" CACHED_SIGNATURE_FILE="${DIB_IMAGE_CACHE}/${FILENAME_BASE}.asc" @@ -89,7 +63,7 @@ else # this key can be verified at one of the following places # https://wiki.gentoo.org/wiki/Project:RelEng#Keys # https://dev.gentoo.org/~dolsen/releases/keyrings/gentoo-keys-*.tar.xz - # http://distfiles.gentoo.org/distfiles/gentoo-keys-*.tar.xz + # https://distfiles.gentoo.org/distfiles/gentoo-keys-*.tar.xz # check the sig file if ! gpgv --keyring "${TMP_HOOKS_PATH}"/extra-data.d/gentoo-releng.gpg "${CACHED_SIGNATURE_FILE}" "${CACHED_FILE}"; then echo 'invalid signature file' @@ -110,3 +84,4 @@ sudo tar -C "${TARGET_ROOT}" --numeric-owner --xattrs -xf "${CACHED_FILE}" # This broken link confuses things like dhclient. # [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 echo -e "# This file intentionally left blank\n" | sudo tee "${TARGET_ROOT}"/etc/resolv.conf + diff --git a/diskimage_builder/elements/growroot/init-scripts/openrc/growroot b/diskimage_builder/elements/growroot/init-scripts/openrc/growroot index 4656fbcf7..1783b6124 100755 --- a/diskimage_builder/elements/growroot/init-scripts/openrc/growroot +++ b/diskimage_builder/elements/growroot/init-scripts/openrc/growroot @@ -1,4 +1,4 @@ -#!/sbin/runscript +#!/sbin/openrc-run start() { /usr/local/sbin/growroot diff --git a/diskimage_builder/elements/install-static/pkg-map b/diskimage_builder/elements/install-static/pkg-map new file mode 100644 index 000000000..2445bc098 --- /dev/null +++ b/diskimage_builder/elements/install-static/pkg-map @@ -0,0 +1,7 @@ +{ + "family":{ + "gentoo": { + "rsync": "net-misc/rsync" + } + } +} diff --git a/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml b/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml new file mode 100644 index 000000000..4a71ba3d6 --- /dev/null +++ b/releasenotes/notes/gentoo-profile-23.0-99357c919639bd3f.yaml @@ -0,0 +1,14 @@ +features: + - Supports Gentoo profile 23.0 and removes support for the nonworking + 17.1 and 17.0 profiles. + - Gentoo element updated to avoid using testing (~arch) packages. + - Gentoo element now uses upstream binary package host by default. +fixes: + - Fixed an issue where the growroot element on openrc init systems would + not function. + - Fixed an issue where the devuser element was unable to grant sudo + capabilities on gentoo images. + - Fixed an issue in Gentoo implmentation for install-packages element + where build time would grow linearly with each additional package removal. + Now, all removed packages are deselected and removed in a single + transaction.