devstack/inc/python
Clark Boylan 065779517f Properly get pip version
The old code was strip()ing the version string instead of split()ing the
version string so we always got the first character of the version
string. This worked fine as long as the pip version was single digit but
as soon as it rolls over to '10.stuff' we will compare:

  pip version 1 (instead of 10) > 6

Which fails bceause 1 is less than six. Instaed we really do want to
compare 10 > 6 so use split on '.' instead.

Change-Id: Ic7d0c04d7fa77774ab2d70fb9d11f182becec553
2017-10-20 12:16:32 -07:00

618 lines
20 KiB
Bash

#!/bin/bash
#
# **inc/python** - Python-related functions
#
# Support for pip/setuptools interfaces and virtual environments
#
# External functions used:
# - GetOSVersion
# - is_fedora
# - is_suse
# - safe_chown
# Save trace setting
INC_PY_TRACE=$(set +o | grep xtrace)
set +o xtrace
# Global Config Variables
# PROJECT_VENV contains the name of the virtual environment for each
# project. A null value installs to the system Python directories.
declare -A -g PROJECT_VENV
# Python Functions
# ================
# Get the path to the pip command.
# get_pip_command
function get_pip_command {
local version="$1"
# NOTE(dhellmann): I don't know if we actually get a pip3.4-python
# under any circumstances.
which pip${version} || which pip${version}-python
if [ $? -ne 0 ]; then
die $LINENO "Unable to find pip${version}; cannot continue"
fi
}
# Get the path to the directory where python executables are installed.
# get_python_exec_prefix
function get_python_exec_prefix {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
$xtrace
if is_fedora || is_suse; then
echo "/usr/bin"
else
echo "/usr/local/bin"
fi
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr packagename
function pip_install_gr {
local name=$1
local clean_name
clean_name=$(get_from_global_requirements $name)
pip_install $clean_name
}
# Wrapper for ``pip install`` that only installs versions of libraries
# from the global-requirements specification with extras.
#
# Uses globals ``REQUIREMENTS_DIR``
#
# pip_install_gr_extras packagename extra1,extra2,...
function pip_install_gr_extras {
local name=$1
local extras=$2
local clean_name
clean_name=$(get_from_global_requirements $name)
pip_install $clean_name[$extras]
}
# Determine the python versions supported by a package
function get_python_versions_for_package {
local name=$1
cd $name && python setup.py --classifiers \
| grep 'Language' | cut -f5 -d: | grep '\.' | tr '\n' ' '
}
# Check for python3 classifier in local directory
function check_python3_support_for_package_local {
local name=$1
cd $name
set +e
classifier=$(python setup.py --classifiers \
| grep 'Programming Language :: Python :: 3$')
set -e
echo $classifier
}
# Check for python3 classifier on pypi
function check_python3_support_for_package_remote {
local name=$1
set +e
classifier=$(curl -s -L "https://pypi.python.org/pypi/$name/json" \
| grep '"Programming Language :: Python :: 3"')
set -e
echo $classifier
}
# python3_enabled_for() checks if the service(s) specified as arguments are
# enabled by the user in ``ENABLED_PYTHON3_PACKAGES``.
#
# Multiple services specified as arguments are ``OR``'ed together; the test
# is a short-circuit boolean, i.e it returns on the first match.
#
# Uses global ``ENABLED_PYTHON3_PACKAGES``
# python3_enabled_for dir [dir ...]
function python3_enabled_for {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local enabled=1
local dirs=$@
local dir
for dir in ${dirs}; do
[[ ,${ENABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0
done
$xtrace
return $enabled
}
# python3_disabled_for() checks if the service(s) specified as arguments are
# disabled by the user in ``DISABLED_PYTHON3_PACKAGES``.
#
# Multiple services specified as arguments are ``OR``'ed together; the test
# is a short-circuit boolean, i.e it returns on the first match.
#
# Uses global ``DISABLED_PYTHON3_PACKAGES``
# python3_disabled_for dir [dir ...]
function python3_disabled_for {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local enabled=1
local dirs=$@
local dir
for dir in ${dirs}; do
[[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]] && enabled=0
done
$xtrace
return $enabled
}
# enable_python3_package() adds the repositories passed as argument to the
# ``ENABLED_PYTHON3_PACKAGES`` list, if they are not already present.
#
# For example:
# enable_python3_package nova
#
# Uses global ``ENABLED_PYTHON3_PACKAGES``
# enable_python3_package dir [dir ...]
function enable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local tmpsvcs="${ENABLED_PYTHON3_PACKAGES}"
local python3
for dir in $@; do
if [[ ,${DISABLED_PYTHON3_PACKAGES}, =~ ,${dir}, ]]; then
warn $LINENO "Attempt to enable_python3_package ${dir} when it has been disabled"
continue
fi
if ! python3_enabled_for $dir; then
tmpsvcs+=",$dir"
fi
done
ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$tmpsvcs")
$xtrace
}
# disable_python3_package() prepares the services passed as argument to be
# removed from the ``ENABLED_PYTHON3_PACKAGES`` list, if they are present.
#
# For example:
# disable_python3_package swift
#
# Uses globals ``ENABLED_PYTHON3_PACKAGES`` and ``DISABLED_PYTHON3_PACKAGES``
# disable_python3_package dir [dir ...]
function disable_python3_package {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local disabled_svcs="${DISABLED_PYTHON3_PACKAGES}"
local enabled_svcs=",${ENABLED_PYTHON3_PACKAGES},"
local dir
for dir in $@; do
disabled_svcs+=",$dir"
if python3_enabled_for $dir; then
enabled_svcs=${enabled_svcs//,$dir,/,}
fi
done
DISABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$disabled_svcs")
ENABLED_PYTHON3_PACKAGES=$(_cleanup_service_list "$enabled_svcs")
$xtrace
}
# Wrapper for ``pip install`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``PIP_VIRTUAL_ENV``,
# ``PIP_UPGRADE``, ``TRACK_DEPENDS``, ``*_proxy``,
# Usage:
# pip_install pip_arguments
function pip_install {
local xtrace result
xtrace=$(set +o | grep xtrace)
set +o xtrace
local upgrade=""
local offline=${OFFLINE:-False}
if [[ "$offline" == "True" || -z "$@" ]]; then
$xtrace
return
fi
time_start "pip_install"
PIP_UPGRADE=$(trueorfalse False PIP_UPGRADE)
if [[ "$PIP_UPGRADE" = "True" ]] ; then
upgrade="--upgrade"
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
# Try to extract the path of the package we are installing into
# package_dir. We need this to check for test-requirements.txt,
# at least.
#
# ${!#} expands to the last positional argument to this function.
# With "extras" syntax included, our arguments might be something
# like:
# -e /path/to/fooproject[extra]
# Thus this magic line grabs just the path without extras
#
# Note that this makes no sense if this is a pypi (rather than
# local path) install; ergo you must check this path exists before
# use. Also, if we had multiple or mixed installs, we would also
# likely break. But for historical reasons, it's basically only
# the other wrapper functions in here calling this to install
# local packages, and they do so with single call per install. So
# this works (for now...)
local package_dir=${!#%\[*\]}
if [[ $TRACK_DEPENDS = True && ! "$@" =~ virtualenv ]]; then
# TRACK_DEPENDS=True installation creates a circular dependency when
# we attempt to install virtualenv into a virtualenv, so we must global
# that installation.
source $DEST/.venv/bin/activate
local cmd_pip=$DEST/.venv/bin/pip
local sudo_pip="env"
else
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
else
local cmd_pip
cmd_pip=$(get_pip_command $PYTHON2_VERSION)
local sudo_pip="sudo -H"
if python3_enabled; then
# Look at the package classifiers to find the python
# versions supported, and if we find the version of
# python3 we've been told to use, use that instead of the
# default pip
local python_versions
# Special case some services that have experimental
# support for python3 in progress, but don't claim support
# in their classifier
echo "Check python version for : $package_dir"
if python3_disabled_for ${package_dir##*/}; then
echo "Explicitly using $PYTHON2_VERSION version to install $package_dir based on DISABLED_PYTHON3_PACKAGES"
elif python3_enabled_for ${package_dir##*/}; then
echo "Explicitly using $PYTHON3_VERSION version to install $package_dir based on ENABLED_PYTHON3_PACKAGES"
sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
cmd_pip=$(get_pip_command $PYTHON3_VERSION)
elif [[ -d "$package_dir" ]]; then
python_versions=$(get_python_versions_for_package $package_dir)
if [[ $python_versions =~ $PYTHON3_VERSION ]]; then
echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on classifiers"
sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
cmd_pip=$(get_pip_command $PYTHON3_VERSION)
else
# The package may not have yet advertised python3.5
# support so check for just python3 classifier and log
# a warning.
python3_classifier=$(check_python3_support_for_package_local $package_dir)
if [[ ! -z "$python3_classifier" ]]; then
echo "Automatically using $PYTHON3_VERSION version to install $package_dir based on local package settings"
sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
cmd_pip=$(get_pip_command $PYTHON3_VERSION)
fi
fi
else
# Check pypi as we don't have the package on disk
package=$(echo $package_dir | grep -o '^[.a-zA-Z0-9_-]*')
python3_classifier=$(check_python3_support_for_package_remote $package)
if [[ ! -z "$python3_classifier" ]]; then
echo "Automatically using $PYTHON3_VERSION version to install $package based on remote package settings"
sudo_pip="$sudo_pip LC_ALL=en_US.UTF-8"
cmd_pip=$(get_pip_command $PYTHON3_VERSION)
fi
fi
fi
fi
fi
cmd_pip="$cmd_pip install"
# Always apply constraints
cmd_pip="$cmd_pip -c $REQUIREMENTS_DIR/upper-constraints.txt"
# FIXME(dhellmann): Need to force multiple versions of pip for
# packages like setuptools?
local pip_version
pip_version=$(python -c "import pip; \
print(pip.__version__.split('.')[0])")
if (( pip_version<6 )); then
die $LINENO "Currently installed pip version ${pip_version} does not" \
"meet minimum requirements (>=6)."
fi
$xtrace
# Also install test requirements
local install_test_reqs=""
local test_req="${package_dir}/test-requirements.txt"
if [[ -e "$test_req" ]]; then
install_test_reqs="-r $test_req"
fi
# adding SETUPTOOLS_SYS_PATH_TECHNIQUE is a workaround to keep
# the same behaviour of setuptools before version 25.0.0.
# related issue: https://github.com/pypa/pip/issues/3874
$sudo_pip \
http_proxy="${http_proxy:-}" \
https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
PIP_FIND_LINKS=$PIP_FIND_LINKS \
SETUPTOOLS_SYS_PATH_TECHNIQUE=rewrite \
$cmd_pip $upgrade $install_test_reqs \
$@
result=$?
time_stop "pip_install"
return $result
}
function pip_uninstall {
# Skip uninstall if offline
[[ "${OFFLINE}" = "True" ]] && return
local name=$1
if [[ -n ${PIP_VIRTUAL_ENV:=} && -d ${PIP_VIRTUAL_ENV} ]]; then
local cmd_pip=$PIP_VIRTUAL_ENV/bin/pip
local sudo_pip="env"
else
local cmd_pip
cmd_pip=$(get_pip_command $PYTHON2_VERSION)
local sudo_pip="sudo -H"
fi
# don't error if we can't uninstall, it might not be there
$sudo_pip $cmd_pip uninstall -y $name || /bin/true
}
# get version of a package from global requirements file
# get_from_global_requirements <package>
function get_from_global_requirements {
local package=$1
local required_pkg
required_pkg=$(grep -i -h ^${package} $REQUIREMENTS_DIR/global-requirements.txt | cut -d\# -f1)
if [[ $required_pkg == "" ]]; then
die $LINENO "Can't find package $package in requirements"
fi
echo $required_pkg
}
# should we use this library from their git repo, or should we let it
# get pulled in via pip dependencies.
function use_library_from_git {
local name=$1
local enabled=1
[[ ${LIBS_FROM_GIT} = 'ALL' ]] || [[ ,${LIBS_FROM_GIT}, =~ ,${name}, ]] && enabled=0
return $enabled
}
# determine if a package was installed from git
function lib_installed_from_git {
local name=$1
# Note "pip freeze" doesn't always work here, because it tries to
# be smart about finding the remote of the git repo the package
# was installed from. This doesn't work with zuul which clones
# repos with no remote.
#
# The best option seems to be to use "pip list" which will tell
# you the path an editable install was installed from; for example
# in response to something like
# pip install -e 'git+http://git.openstack.org/openstack-dev/bashate#egg=bashate'
# pip list shows
# bashate (0.5.2.dev19, /tmp/env/src/bashate)
# Thus we look for "path after a comma" to indicate we were
# installed from some local place
pip list 2>/dev/null | grep -- "$name" | grep -q -- ', .*)$'
}
# check that everything that's in LIBS_FROM_GIT was actually installed
# correctly, this helps double check issues with library fat fingering.
function check_libs_from_git {
local lib=""
local not_installed=""
for lib in $(echo ${LIBS_FROM_GIT} | tr "," " "); do
if ! lib_installed_from_git "$lib"; then
not_installed+=" $lib"
fi
done
# if anything is not installed, say what it is.
if [[ -n "$not_installed" ]]; then
die $LINENO "The following LIBS_FROM_GIT were not installed correct: $not_installed"
fi
}
# setup a library by name. If we are trying to use the library from
# git, we'll do a git based install, otherwise we'll punt and the
# library should be installed by a requirements pull from another
# project.
function setup_lib {
local name=$1
local dir=${GITDIR[$name]}
setup_install $dir
}
# setup a library by name in editable mode. If we are trying to use
# the library from git, we'll do a git based install, otherwise we'll
# punt and the library should be installed by a requirements pull from
# another project.
#
# use this for non namespaced libraries
function setup_dev_lib {
local name=$1
local dir=${GITDIR[$name]}
if python3_enabled; then
# Turn off Python 3 mode and install the package again,
# forcing a Python 2 installation. This ensures that all libs
# being used for development are installed under both versions
# of Python.
echo "Installing $name again without Python 3 enabled"
USE_PYTHON3=False
setup_develop $dir
USE_PYTHON3=True
fi
setup_develop $dir
}
# this should be used if you want to install globally, all libraries should
# use this, especially *oslo* ones
#
# setup_install project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install <project_dir>[<extras>]"
function setup_install {
local project_dir=$1
local extras=$2
_setup_package_with_constraints_edit $project_dir "" $extras
}
# this should be used for projects which run services, like all services
#
# setup_develop project_dir [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install -e <project_dir>[<extras>]"
function setup_develop {
local project_dir=$1
local extras=$2
_setup_package_with_constraints_edit $project_dir -e $extras
}
# determine if a project as specified by directory is in
# projects.txt. This will not be an exact match because we throw away
# the namespacing when we clone, but it should be good enough in all
# practical ways.
function is_in_projects_txt {
local project_dir=$1
local project_name
project_name=$(basename $project_dir)
grep -q "/$project_name\$" $REQUIREMENTS_DIR/projects.txt
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`
#
# Updates the constraints from REQUIREMENTS_DIR to reflect the
# future installed state of this package. This ensures when we
# install this package we get the from source version.
#
# Uses globals ``REQUIREMENTS_DIR``
# _setup_package_with_constraints_edit project_dir flags [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# flags: pip CLI options/flags
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install <flags> <project_dir>[<extras>]"
function _setup_package_with_constraints_edit {
local project_dir=$1
local flags=$2
local extras=$3
# Normalize the directory name to avoid
# "installation from path or url cannot be constrained to a version"
# error.
# REVISIT(yamamoto): Remove this when fixed in pip.
# https://github.com/pypa/pip/pull/3582
project_dir=$(cd $project_dir && pwd)
if [ -n "$REQUIREMENTS_DIR" ]; then
# Constrain this package to this project directory from here on out.
local name
name=$(awk '/^name.*=/ {print $3}' $project_dir/setup.cfg)
$REQUIREMENTS_DIR/.venv/bin/edit-constraints \
$REQUIREMENTS_DIR/upper-constraints.txt -- $name \
"$flags file://$project_dir#egg=$name"
fi
setup_package $project_dir "$flags" $extras
}
# ``pip install -e`` the package, which processes the dependencies
# using pip before running `setup.py develop`
#
# Uses globals ``STACK_USER``
# setup_package project_dir [flags] [extras]
# project_dir: directory of project repo (e.g., /opt/stack/keystone)
# flags: pip CLI options/flags
# extras: comma-separated list of optional dependencies to install
# (e.g., ldap,memcache).
# See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
# The command is like "pip install <flags> <project_dir>[<extras>]"
function setup_package {
local project_dir=$1
local flags=$2
local extras=$3
# if the flags variable exists, and it doesn't look like a flag,
# assume it's actually the extras list.
if [[ -n "$flags" && -z "$extras" && ! "$flags" =~ ^-.* ]]; then
extras=$flags
flags=""
fi
if [[ ! -z "$extras" ]]; then
extras="[$extras]"
fi
pip_install $flags "$project_dir$extras"
# ensure that further actions can do things like setup.py sdist
if [[ "$flags" == "-e" ]]; then
safe_chown -R $STACK_USER $1/*.egg-info
fi
}
# Report whether python 3 should be used
function python3_enabled {
if [[ $USE_PYTHON3 == "True" ]]; then
return 0
else
return 1
fi
}
# Install python3 packages
function install_python3 {
if is_ubuntu; then
apt_get install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev
elif is_suse; then
install_package python3-devel python3-dbm
fi
}
function install_devstack_tools {
# intentionally old to ensure devstack-gate has control
local dstools_version=${DSTOOLS_VERSION:-0.1.2}
install_python3
sudo pip3 install -U devstack-tools==${dstools_version}
}
# Restore xtrace
$INC_PY_TRACE
# Local variables:
# mode: shell-script
# End: