From fa9aadfdd8c5f67a47f5a4abafbae0671283affa Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 15 Jan 2019 18:31:05 +1100
Subject: [PATCH] Install from bindep.txt in plugins

This allows plugins to specify their binary dependencies in bindep
format.

Some thinking on the implementation: this is in contrast to the
files/[deb|rpm] installation, which is called from the external
install_prereqs.sh script.  This script being an externally callable
entry-point is really an artifact of the days when we would build
snapshot images for CI and wanted to pre-cache downloads.  These days
we use the mirror system to keep packages close to CI nodes.  Thus
rather than expand install_prereqs.sh to also be installing
virtualenvs and python dependencies, this seems to fit better as a
separate internal phase of stack.sh.

Documentation is updated

Change-Id: Icbdfbf97c17c906a7ae86f43e80eb2c445816228
---
 doc/source/plugins.rst | 56 ++++++++++++++++++++++++++++++++++++------
 functions-common       | 24 ++++++++++++++++++
 stack.sh               |  7 ++++++
 3 files changed, 80 insertions(+), 7 deletions(-)

diff --git a/doc/source/plugins.rst b/doc/source/plugins.rst
index 89b9381813..de7bdf22b0 100644
--- a/doc/source/plugins.rst
+++ b/doc/source/plugins.rst
@@ -222,14 +222,20 @@ dependency mechanism is beyond the scope of the current work.
 System Packages
 ===============
 
-Devstack provides a framework for getting packages installed at an early
-phase of its execution. These packages may be defined in a plugin as files
-that contain new-line separated lists of packages required by the plugin
 
-Supported packaging systems include apt and yum across multiple distributions.
-To enable a plugin to hook into this and install package dependencies, packages
-may be listed at the following locations in the top-level of the plugin
-repository:
+
+Devstack based
+--------------
+
+Devstack provides a custom framework for getting packages installed at
+an early phase of its execution.  These packages may be defined in a
+plugin as files that contain new-line separated lists of packages
+required by the plugin
+
+Supported packaging systems include apt and yum across multiple
+distributions.  To enable a plugin to hook into this and install
+package dependencies, packages may be listed at the following
+locations in the top-level of the plugin repository:
 
 - ``./devstack/files/debs/$plugin_name`` - Packages to install when running
   on Ubuntu, Debian or Linux Mint.
@@ -240,6 +246,42 @@ repository:
 - ``./devstack/files/rpms-suse/$plugin_name`` - Packages to install when
   running on SUSE Linux or openSUSE.
 
+Although there a no plans to remove this method of installing
+packages, plugins should consider it deprecated for ``bindep`` support
+described below.
+
+bindep
+------
+
+The `bindep <https://docs.openstack.org/infra/bindep>`__ project has
+become the defacto standard for OpenStack projects to specify binary
+dependencies.
+
+A plugin may provide a ``./devstack/files/bindep.txt`` file, which
+will be called with the *default* profile to install packages.  For
+details on the syntax, etc. see the bindep documentation.
+
+It is also possible to use the ``bindep.txt`` of projects that are
+being installed from source with the ``-bindep`` flag available in
+install functions.  For example
+
+.. code-block:: bash
+
+  if use_library_from_git "diskimage-builder"; then
+     GITREPO["diskimage-builder"]=$DISKIMAGE_BUILDER_REPO_URL
+     GITDIR["diskimage-builder"]=$DEST/diskimage-builder
+     GITBRANCH["diskimage-builder"]=$DISKIMAGE_BUILDER_REPO_REF
+     git_clone_by_name "diskimage-builder"
+     setup_dev_lib -bindep "diskimage-builder"
+  fi
+
+will result in any packages required by the ``bindep.txt`` of the
+``diskimage-builder`` project being installed.  Note however that jobs
+that switch projects between source and released/pypi installs
+(e.g. with a ``foo-dsvm`` and a ``foo-dsvm-src`` test to cover both
+released dependencies and master versions) will have to deal with
+``bindep.txt`` being unavailable without the source directory.
+
 
 Using Plugins in the OpenStack Gate
 ===================================
diff --git a/functions-common b/functions-common
index 6651cc2661..2a5d13998a 100644
--- a/functions-common
+++ b/functions-common
@@ -1248,6 +1248,30 @@ function get_plugin_packages {
     $xtrace
 }
 
+# Search plugins for a bindep.txt file
+#
+# Uses globals ``BINDEP_CMD``, ``GITDIR``, ``DEVSTACK_PLUGINS``
+#
+# Note this is only valid after BINDEP_CMD is setup in stack.sh, and
+# is thus not really intended to be called externally.
+function _get_plugin_bindep_packages {
+    local xtrace
+    xtrace=$(set +o | grep xtrace)
+    set +o xtrace
+
+    local bindep_file
+    local packages
+
+    for plugin in ${DEVSTACK_PLUGINS//,/ }; do
+        bindep_file=${GITDIR[$plugin]}/devstack/files/bindep.txt
+        if [[ -f ${bindep_file} ]]; then
+            packages+=$($BINDEP_CMD -b --file ${bindep_file} || true)
+        fi
+    done
+    echo "${packages}"
+    $xtrace
+}
+
 # Distro-agnostic package installer
 # Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
 # install_package package [package ...]
diff --git a/stack.sh b/stack.sh
index ad1359e926..95e4df4277 100755
--- a/stack.sh
+++ b/stack.sh
@@ -805,6 +805,13 @@ install_infra
 $VIRTUALENV_CMD $DEST/bindep-venv
 # TODO(ianw) : optionally install from zuul checkout?
 $DEST/bindep-venv/bin/pip install bindep
+export BINDEP_CMD=${DEST}/bindep-venv/bin/bindep
+
+# Install packages as defined in plugin bindep.txt files
+pkgs="$( _get_plugin_bindep_packages )"
+if [[ -n "${pkgs}" ]]; then
+    install_package ${pkgs}
+fi
 
 # Extras Pre-install
 # ------------------