Add options for development bindep install

This adds a -bindep option to the key development library install
functions.  With this option the bindep.txt file will be referenced
and the relevant packages installed.

Change-Id: I856f1f59fca49b6020920d8f859b797f3b904300

functions-common

@@ -1381,6 +1381,35 @@ function zypper_install {
         zypper --non-interactive install --auto-agree-with-licenses --no-recommends "$@"
 }
 
+# Run bindep and install packages it outputs
+#
+# Usage:
+#  install_bindep <path-to-bindep.txt> [profile,profile]
+#
+# Note unlike the bindep command itself, profile(s) specified should
+# be a single, comma-separated string, no spaces.
+function install_bindep {
+    local file=$1
+    local profiles=${2:-""}
+    local pkgs
+
+    if [[ ! -f $file ]]; then
+        die $LINENO "Can not find bindep file: $file"
+    fi
+
+    # converting here makes it much easier to work with passing
+    # arguments
+    profiles=${profiles/,/ /}
+
+    # Note bindep returns 1 when packages need to be installed, so we
+    # have to ignore it's return for "-e"
+    pkgs=$($DEST/bindep-venv/bin/bindep -b --file $file $profiles || true)
+
+    if [[ -n "${pkgs}" ]]; then
+        install_package ${pkgs}
+    fi
+}
+
 function write_user_unit_file {
     local service=$1
     local command="$2"

inc/python

@@ -445,7 +445,14 @@ function setup_lib {
 # another project.
 #
 # use this for non namespaced libraries
+#
+# setup_dev_lib [-bindep] <name>
 function setup_dev_lib {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local name=$1
     local dir=${GITDIR[$name]}
     if python3_enabled; then
@@ -455,10 +462,10 @@ function setup_dev_lib {
         # of Python.
         echo "Installing $name again without Python 3 enabled"
         USE_PYTHON3=False
-        setup_develop $dir
+        setup_develop $bindep $dir
         USE_PYTHON3=True
     fi
-    setup_develop $dir
+    setup_develop $bindep $dir
 }
 
 # this should be used if you want to install globally, all libraries should
@@ -469,11 +476,17 @@ function setup_dev_lib {
 # extras: comma-separated list of optional dependencies to install
 #         (e.g., ldap,memcache).
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
+# bindep: Set "-bindep" as first argument to install bindep.txt packages
 # The command is like "pip install <project_dir>[<extras>]"
 function setup_install {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local extras=$2
-    _setup_package_with_constraints_edit $project_dir "" $extras
+    _setup_package_with_constraints_edit $bindep $project_dir "" $extras
 }
 
 # this should be used for projects which run services, like all services
@@ -485,9 +498,14 @@ function setup_install {
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install -e <project_dir>[<extras>]"
 function setup_develop {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local extras=$2
-    _setup_package_with_constraints_edit $project_dir -e $extras
+    _setup_package_with_constraints_edit $bindep $project_dir -e $extras
 }
 
 # ``pip install -e`` the package, which processes the dependencies
@@ -506,6 +524,11 @@ function setup_develop {
 #         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 # The command is like "pip install <flags> <project_dir>[<extras>]"
 function _setup_package_with_constraints_edit {
+    local bindep
+    if [[ $1 == -bindep* ]]; then
+        bindep="${1}"
+        shift
+    fi
     local project_dir=$1
     local flags=$2
     local extras=$3
@@ -526,7 +549,7 @@ function _setup_package_with_constraints_edit {
             "$flags file://$project_dir#egg=$name"
     fi
 
-    setup_package $project_dir "$flags" $extras
+    setup_package $bindep $project_dir "$flags" $extras
 
     # If this project is in LIBS_FROM_GIT, verify it was actually installed
     # correctly.  This helps catch errors caused by constraints mismatches.
@@ -538,17 +561,30 @@ function _setup_package_with_constraints_edit {
 }
 
 # ``pip install -e`` the package, which processes the dependencies
-# using pip before running `setup.py develop`
+# using pip before running `setup.py develop`.  The command is like
+# "pip install <flags> <project_dir>[<extras>]"
 #
 # Uses globals ``STACK_USER``
-# setup_package project_dir [flags] [extras]
+#
-# project_dir: directory of project repo (e.g., /opt/stack/keystone)
+# Usage:
-# flags: pip CLI options/flags
+#  setup_package [-bindep[=profile,profile]] <project_dir> <flags> [extras]
-# extras: comma-separated list of optional dependencies to install
+#
-#         (e.g., ldap,memcache).
+# -bindep     : Use bindep to install dependencies; select extra profiles
-#         See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
+#               as comma separated arguments after "="
-# The command is like "pip install <flags> <project_dir>[<extras>]"
+# project_dir : directory of project repo (e.g., /opt/stack/keystone)
+# flags       : pip CLI options/flags
+# extras      : comma-separated list of optional dependencies to install
+#               (e.g., ldap,memcache).
+#               See https://docs.openstack.org/pbr/latest/user/using.html#extra-requirements
 function setup_package {
+    local bindep=0
+    local bindep_flag=""
+    local bindep_profiles=""
+    if [[ $1 == -bindep* ]]; then
+        bindep=1
+        IFS="=" read bindep_flag bindep_profiles <<< ${1}
+        shift
+    fi
     local project_dir=$1
     local flags=$2
     local extras=$3
@@ -564,6 +600,11 @@ function setup_package {
         extras="[$extras]"
     fi
 
+    # install any bindep packages
+    if [[ $bindep == 1 ]]; then
+        install_bindep $project_dir/bindep.txt $bindep_profiles
+    fi
+
     pip_install $flags "$project_dir$extras"
     # ensure that further actions can do things like setup.py sdist
     if [[ "$flags" == "-e" ]]; then

stack.sh

@@ -801,6 +801,11 @@ fi
 # Install required infra support libraries
 install_infra
 
+# Install bindep
+$VIRTUALENV_CMD $DEST/bindep-venv
+# TODO(ianw) : optionally install from zuul checkout?
+$DEST/bindep-venv/bin/pip install bindep
+
 # Extras Pre-install
 # ------------------
 # Phase: pre-install

stackrc

@@ -149,6 +149,13 @@ export PYTHON3_VERSION=${PYTHON3_VERSION:-${_DEFAULT_PYTHON3_VERSION:-3.5}}
 _DEFAULT_PYTHON2_VERSION="$(_get_python_version python2)"
 export PYTHON2_VERSION=${PYTHON2_VERSION:-${_DEFAULT_PYTHON2_VERSION:-2.7}}
 
+# Create a virtualenv with this
+if [[ ${USE_PYTHON3} == True ]]; then
+    export VIRTUALENV_CMD="python3 -m venv"
+else
+    export VIRTUALENV_CMD="virtualenv "
+fi
+
 # allow local overrides of env variables, including repo config
 if [[ -f $RC_DIR/localrc ]]; then
     # Old-style user-supplied config