From a40f9cb91fbedddec89f0ffd6c7dd4b3828a232e Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 4 Apr 2018 14:02:30 -0700 Subject: [PATCH] Add option to install everything in global venvs Since we are python3 only for openstack we create a single python3 virtualenv to install all the packages into. This gives us the benefits of installing into a virtualenv while still ensuring coinstallability. This is a major change and will likely break many things. There are several reasons for this. The change that started this effort was pip stopped uninstalling packages which used distutils to generate their package installation. Many distro packages do this which meant that pip installed packages and distro packages could not coexist in the global install space. More recently git has made pip installing repos as root more difficult due to file ownership concerns. Currently the switch to the global venv is optional, but if we go down this path we should very quickly remove the old global installation method as it has only caused us problems. Major hurdles we have to get over are convincing rootwrap to trust binaries in the virtualenvs (so you'll notice we update rootwrap configs). Some distros still have issues, keep them using the old setup for now. Depends-On: https://review.opendev.org/c/openstack/grenade/+/880266 Co-Authored-By: Dr. Jens Harbott Change-Id: If9bc7ba45522189d03f19b86cb681bb150ee2f25 --- .zuul.yaml | 8 ++++++++ files/apache-horizon.template | 1 + functions-common | 5 +++++ inc/python | 34 +++++++++++++++++++++++++++++++--- inc/rootwrap | 5 +++++ lib/glance | 3 +++ lib/horizon | 6 ++++++ lib/tls | 7 +++++-- stack.sh | 12 ++++++++++++ stackrc | 8 ++++++++ tools/install_prereqs.sh | 2 ++ tools/memory_tracker.sh | 7 ++++++- 12 files changed, 92 insertions(+), 6 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index a7be67153b..803db3a3fa 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -700,6 +700,9 @@ # TODO(kopecmartin) n-v until the following is resolved: # https://bugs.launchpad.net/neutron/+bug/1979047 voting: false + vars: + devstack_localrc: + GLOBAL_VENV: false - job: name: devstack-platform-debian-bullseye @@ -709,6 +712,9 @@ timeout: 9000 vars: configure_swap_size: 4096 + devstack_localrc: + # TODO(frickler): drop this once wheel build is fixed + MYSQL_GATHER_PERFORMANCE: false - job: name: devstack-platform-rocky-blue-onyx @@ -718,6 +724,8 @@ timeout: 9000 vars: configure_swap_size: 4096 + devstack_localrc: + GLOBAL_VENV: false - job: name: devstack-platform-ubuntu-focal diff --git a/files/apache-horizon.template b/files/apache-horizon.template index efcfc0360b..da7a7d26c3 100644 --- a/files/apache-horizon.template +++ b/files/apache-horizon.template @@ -39,4 +39,5 @@ CustomLog /var/log/%APACHE_NAME%/horizon_access.log combined +%WSGIPYTHONHOME% WSGISocketPrefix /var/run/%APACHE_NAME% diff --git a/functions-common b/functions-common index 5e1aa34279..f752271976 100644 --- a/functions-common +++ b/functions-common @@ -1522,6 +1522,7 @@ function write_user_unit_file { mkdir -p $SYSTEMD_DIR iniset -sudo $unitfile "Unit" "Description" "Devstack $service" + iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\"" iniset -sudo $unitfile "Service" "User" "$user" iniset -sudo $unitfile "Service" "ExecStart" "$command" iniset -sudo $unitfile "Service" "KillMode" "process" @@ -1549,6 +1550,7 @@ function write_uwsgi_user_unit_file { mkdir -p $SYSTEMD_DIR iniset -sudo $unitfile "Unit" "Description" "Devstack $service" + iniset -sudo $unitfile "Service" "Environment" "\"PATH=$PATH\"" iniset -sudo $unitfile "Service" "SyslogIdentifier" "$service" iniset -sudo $unitfile "Service" "User" "$user" iniset -sudo $unitfile "Service" "ExecStart" "$command" @@ -1614,6 +1616,9 @@ function _run_under_systemd { fi local env_vars="$5" if [[ "$command" =~ "uwsgi" ]] ; then + if [[ "$GLOBAL_VENV" == "True" ]] ; then + cmd="$cmd --venv $DEVSTACK_VENV" + fi write_uwsgi_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars" else write_user_unit_file $systemd_service "$cmd" "$group" "$user" "$env_vars" diff --git a/inc/python b/inc/python index a24f4e910a..cc6e01fede 100644 --- a/inc/python +++ b/inc/python @@ -32,6 +32,23 @@ function join_extras { # Python Functions # ================ +# Setup the global devstack virtualenvs and the associated environment +# updates. +function setup_devstack_virtualenv { + # We run devstack out of a global virtualenv. + if [[ ! -d $DEVSTACK_VENV ]] ; then + # Using system site packages to enable nova to use libguestfs. + # This package is currently installed via the distro and not + # available on pypi. + python$PYTHON3_VERSION -m venv --system-site-packages $DEVSTACK_VENV + pip_install -U pip + fi + if [[ ":$PATH:" != *":$DEVSTACK_VENV/bin:"* ]] ; then + export PATH="$DEVSTACK_VENV/bin:$PATH" + export PYTHON="$DEVSTACK_VENV/bin/python3" + fi +} + # Get the path to the pip command. # get_pip_command function get_pip_command { @@ -60,8 +77,11 @@ function get_python_exec_prefix { fi $xtrace - local PYTHON_PATH=/usr/local/bin - echo $PYTHON_PATH + if [[ "$GLOBAL_VENV" == "True" ]] ; then + echo "$DEVSTACK_VENV/bin" + else + echo "/usr/local/bin" + fi } # Wrapper for ``pip install`` that only installs versions of libraries @@ -166,6 +186,14 @@ function pip_install { if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip local sudo_pip="env" + elif [[ "${GLOBAL_VENV}" == "True" && -d ${DEVSTACK_VENV} ]] ; then + # We have to check that the DEVSTACK_VENV exists because early + # devstack boostrapping needs to operate in a system context + # too bootstrap pip. Once pip is bootstrapped we create the + # global venv and can start to use it. + local cmd_pip=$DEVSTACK_VENV/bin/pip + local sudo_pip="env" + echo "Using python $PYTHON3_VERSION to install $package_dir" else local cmd_pip="python$PYTHON3_VERSION -m pip" # See @@ -439,7 +467,7 @@ function setup_package { pip_install $flags "$project_dir$extras" # ensure that further actions can do things like setup.py sdist - if [[ "$flags" == "-e" ]]; then + if [[ "$flags" == "-e" && "$GLOBAL_VENV" == "False" ]]; then safe_chown -R $STACK_USER $1/*.egg-info fi } diff --git a/inc/rootwrap b/inc/rootwrap index 2a6e4b648f..4c65440a4e 100644 --- a/inc/rootwrap +++ b/inc/rootwrap @@ -60,6 +60,11 @@ function configure_rootwrap { sudo install -o root -g root -m 644 $rootwrap_conf_src_dir/rootwrap.conf /etc/${project}/rootwrap.conf sudo sed -e "s:^filters_path=.*$:filters_path=/etc/${project}/rootwrap.d:" -i /etc/${project}/rootwrap.conf + # Rely on $PATH set by devstack to determine what is safe to execute + # by rootwrap rather than use explicit whitelist of paths in + # rootwrap.conf + sudo sed -e 's/^exec_dirs=.*/#&/' -i /etc/${project}/rootwrap.conf + # Set up the rootwrap sudoers local tempfile tempfile=$(mktemp) diff --git a/lib/glance b/lib/glance index 430d94d3a4..e64f00027e 100644 --- a/lib/glance +++ b/lib/glance @@ -47,6 +47,9 @@ USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE) # from CINDER_ENABLED_BACKENDS GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1} GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance +if [[ "$GLOBAL_VENV" == "True" ]] ; then + GLANCE_STORE_ROOTWRAP_BASE_DIR=${DEVSTACK_VENV}/etc/glance +fi # When Cinder is used as a glance store, you can optionally configure cinder to # optimize bootable volume creation by allowing volumes to be cloned directly # in the backend instead of transferring data via Glance. To use this feature, diff --git a/lib/horizon b/lib/horizon index f76f9e557d..611329d619 100644 --- a/lib/horizon +++ b/lib/horizon @@ -115,6 +115,11 @@ function configure_horizon { local horizon_conf horizon_conf=$(apache_site_config_for horizon) + local wsgi_venv_config="" + if [[ "$GLOBAL_VENV" == "True" ]] ; then + wsgi_venv_config="WSGIPythonHome $DEVSTACK_VENV" + fi + # Configure apache to run horizon # Set up the django horizon application to serve via apache/wsgi sudo sh -c "sed -e \" @@ -124,6 +129,7 @@ function configure_horizon { s,%APACHE_NAME%,$APACHE_NAME,g; s,%DEST%,$DEST,g; s,%WEBROOT%,$HORIZON_APACHE_ROOT,g; + s,%WSGIPYTHONHOME%,$wsgi_venv_config,g; \" $FILES/apache-horizon.template >$horizon_conf" if is_ubuntu; then diff --git a/lib/tls b/lib/tls index a1e162d2e2..d35e9e2cee 100644 --- a/lib/tls +++ b/lib/tls @@ -364,8 +364,11 @@ function deploy_int_CA { function fix_system_ca_bundle_path { if is_service_enabled tls-proxy; then local capath - capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass') - + if [[ "$GLOBAL_VENV" == "True" ]] ; then + capath=$($DEVSTACK_VENV/bin/python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass') + else + capath=$(python3 -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass') + fi if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then if is_fedora; then sudo rm -f $capath diff --git a/stack.sh b/stack.sh index ad88eab9d5..c8f7c9d79e 100755 --- a/stack.sh +++ b/stack.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash + # ``stack.sh`` is an opinionated OpenStack developer installation. It # installs and configures various combinations of **Cinder**, **Glance**, # **Horizon**, **Keystone**, **Nova**, **Neutron**, and **Swift** @@ -824,6 +825,17 @@ fi source $TOP_DIR/tools/fixup_stuff.sh fixup_all +if [[ "$GLOBAL_VENV" == "True" ]] ; then + # TODO(frickler): find a better solution for this + sudo ln -sf /opt/stack/data/venv/bin/privsep-helper /usr/local/bin + sudo ln -sf /opt/stack/data/venv/bin/cinder-rtstool /usr/local/bin + sudo ln -sf /opt/stack/data/venv/bin/openstack /usr/local/bin + sudo ln -sf /opt/stack/data/venv/bin/tox /usr/local/bin + sudo ln -sf /opt/stack/data/venv/bin/nova-manage /usr/local/bin + + setup_devstack_virtualenv +fi + # Install subunit for the subunit output stream pip_install -U os-testr diff --git a/stackrc b/stackrc index dcc0ce45e0..0d1880cec9 100644 --- a/stackrc +++ b/stackrc @@ -183,6 +183,14 @@ IDENTITY_API_VERSION=3 # each services ${SERVICE}_ENFORCE_SCOPE variables ENFORCE_SCOPE=$(trueorfalse False ENFORCE_SCOPE) +# Devstack supports the use of a global virtualenv. These variables enable +# and disable this functionality as well as set the path to the virtualenv. +# Note that the DATA_DIR is selected because grenade testing uses a shared +# DATA_DIR but different DEST dirs and we don't want two sets of venvs, +# instead we want one global set. +GLOBAL_VENV=$(trueorfalse True GLOBAL_VENV) +DEVSTACK_VENV=${DEVSTACK_VENV:-$DATA_DIR/venv} + # Enable use of Python virtual environments. Individual project use of # venvs are controlled by the PROJECT_VENV array; every project with # an entry in the array will be installed into the named venv. diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index f2d57c8451..bb470b2927 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -79,6 +79,8 @@ if [[ -n "$SYSLOG" && "$SYSLOG" != "False" ]]; then fi fi +# TODO(clarkb) remove these once we are switched to global venv by default +export PYTHON=$(which python${PYTHON3_VERSION} 2>/dev/null || which python3 2>/dev/null) # Mark end of run # --------------- diff --git a/tools/memory_tracker.sh b/tools/memory_tracker.sh index 6c36534f01..2f404c26fb 100755 --- a/tools/memory_tracker.sh +++ b/tools/memory_tracker.sh @@ -14,7 +14,12 @@ set -o errexit -PYTHON=${PYTHON:-python3} +# TODO(frickler): make this use stackrc variables +if [ -x /opt/stack/data/venv/bin/python ]; then + PYTHON=/opt/stack/data/venv/bin/python +else + PYTHON=${PYTHON:-python3} +fi # time to sleep between checks SLEEP_TIME=20