Implement devstack external plugins

This is an initial pass at plugin infrastructure for devstack which
allows specifying an external repository via:

enable_plugin <name> <giturl> [branch]

It implements the devstack specification for this at
I173dee3d57967b1d2ffd30e4868a2832aeac97ce

Change-Id: I8e4175313b3cf0b12e981122358b1288a7eb0746

doc/source/plugins.rst

@@ -92,6 +92,45 @@ The arguments are:
 -  **clean** - Called by ``clean.sh`` before other services are cleaned,
    but after ``unstack.sh`` has been called.
 
+
+Externally Hosted Plugins
+=========================
+
+Based on the extras.d hooks, DevStack supports a standard mechansim
+for including plugins from external repositories. The plugin interface
+assumes the following:
+
+An external git repository that includes a ``devstack/`` top level
+directory. Inside this directory there can be 2 files.
+
+- ``settings`` - a file containing global variables that will be
+  sourced very early in the process. This is helpful if other plugins
+  might depend on this one, and need access to global variables to do
+  their work.
+- ``plugin.sh`` - the actual plugin. It will be executed by devstack
+  during it's run. The run order will be done in the registration
+  order for these plugins, and will occur immediately after all in
+  tree extras.d dispatch at the phase in question.  The plugin.sh
+  looks like the extras.d dispatcher above **except** it should not
+  include the is_service_enabled conditional. All external plugins are
+  always assumed to be enabled.
+
+Plugins are registered by adding the following to the localrc section
+of ``local.conf``.
+
+They are added in the following format::
+
+  enable_plugin <NAME> <GITURL> [GITREF]
+
+- ``name`` - an arbitrary name. (ex: glustfs, docker, zaqar, congress)
+- ``giturl`` - a valid git url that can be cloned
+- ``gitref`` - an optional git ref (branch / ref / tag) that will be
+  cloned. Defaults to master.
+
+An example would be as follows::
+
+  enable_plugin glusterfs https://github.com/sdague/devstack-plugins glusterfs
+
 Hypervisor
 ==========
 

functions-common

@@ -44,7 +44,6 @@ declare -A GITREPO
 declare -A GITBRANCH
 declare -A GITDIR
 
-
 # Config Functions
 # ================
 
@@ -1722,6 +1721,97 @@ function setup_package {
     fi
 }
 
+# Plugin Functions
+# =================
+
+DEVSTACK_PLUGINS=${DEVSTACK_PLUGINS:-""}
+
+# enable_plugin <name> <url> [branch]
+#
+# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
+# ``url`` is a git url
+# ``branch`` is a gitref. If it's not set, defaults to master
+function enable_plugin {
+    local name=$1
+    local url=$2
+    local branch=${3:-master}
+    DEVSTACK_PLUGINS+=",$name"
+    GITREPO[$name]=$url
+    GITDIR[$name]=$DEST/$name
+    GITBRANCH[$name]=$branch
+}
+
+# fetch_plugins
+#
+# clones all plugins
+function fetch_plugins {
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+
+    # short circuit if nothing to do
+    if [[ -z $plugins ]]; then
+        return
+    fi
+
+    echo "Fetching devstack plugins"
+    for plugin in ${plugins//,/ }; do
+        git_clone_by_name $plugin
+    done
+}
+
+# load_plugin_settings
+#
+# Load settings from plugins in the order that they were registered
+function load_plugin_settings {
+    local plugins="${DEVSTACK_PLUGINS}"
+    local plugin
+
+    # short circuit if nothing to do
+    if [[ -z $plugins ]]; then
+        return
+    fi
+
+    echo "Loading plugin settings"
+    for plugin in ${plugins//,/ }; do
+        local dir=${GITDIR[$plugin]}
+        # source any known settings
+        if [[ -f $dir/devstack/settings ]]; then
+            source $dir/devstack/settings
+        fi
+    done
+}
+
+# run_plugins
+#
+# Run the devstack/plugin.sh in all the plugin directories. These are
+# run in registration order.
+function run_plugins {
+    local mode=$1
+    local phase=$2
+    for plugin in ${plugins//,/ }; do
+        local dir=${GITDIR[$plugin]}
+        if [[ -f $dir/devstack/plugin.sh ]]; then
+            source $dir/devstack/plugin.sh $mode $phase
+        fi
+    done
+}
+
+function run_phase {
+    local mode=$1
+    local phase=$2
+    if [[ -d $TOP_DIR/extras.d ]]; then
+        for i in $TOP_DIR/extras.d/*.sh; do
+            [[ -r $i ]] && source $i $mode $phase
+        done
+    fi
+    # the source phase corresponds to settings loading in plugins
+    if [[ "$mode" == "source" ]]; then
+        load_plugin_settings
+    else
+        run_plugins $mode $phase
+    fi
+}
+
 
 # Service Functions
 # =================

stack.sh

@@ -564,15 +564,14 @@ source $TOP_DIR/lib/neutron
 source $TOP_DIR/lib/ldap
 source $TOP_DIR/lib/dstat
 
+# Clone all external plugins
+fetch_plugins
+
 # Extras Source
 # --------------
 
 # Phase: source
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase source
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i source
-    done
-fi
 
 # Interactive Configuration
 # -------------------------
@@ -714,12 +713,7 @@ source $TOP_DIR/tools/fixup_stuff.sh
 # ------------------
 
 # Phase: pre-install
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase stack pre-install
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i stack pre-install
-    done
-fi
-
 
 install_rpc_backend
 
@@ -865,11 +859,7 @@ fi
 # --------------
 
 # Phase: install
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase stack install
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i stack install
-    done
-fi
 
 if [[ $TRACK_DEPENDS = True ]]; then
     $DEST/.venv/bin/pip freeze > $DEST/requires-post-pip
@@ -1142,11 +1132,7 @@ fi
 # ====================
 
 # Phase: post-config
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase stack post-config
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i stack post-config
-    done
-fi
 
 
 # Local Configuration
@@ -1328,11 +1314,7 @@ merge_config_group $TOP_DIR/local.conf extra
 # ==========
 
 # Phase: extra
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase stack extra
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i stack extra
-    done
-fi
 
 # Local Configuration
 # ===================

unstack.sh

@@ -66,6 +66,8 @@ if [[ -d $TOP_DIR/extras.d ]]; then
     done
 fi
 
+load_plugin_settings
+
 # Determine what system we are running on.  This provides ``os_VENDOR``,
 # ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
 GetOSVersion
@@ -78,11 +80,7 @@ fi
 # ==========
 
 # Phase: unstack
-if [[ -d $TOP_DIR/extras.d ]]; then
+run_phase unstack
-    for i in $TOP_DIR/extras.d/*.sh; do
-        [[ -r $i ]] && source $i unstack
-    done
-fi
 
 if [[ "$Q_USE_DEBUG_COMMAND" == "True" ]]; then
     source $TOP_DIR/openrc