From e3e805186476018b988f68f3ef3dfaccbf3c0f20 Mon Sep 17 00:00:00 2001
From: Clint Adams <clint@gcfm.net>
Date: Thu, 18 Feb 2016 14:46:35 -0500
Subject: [PATCH] Generate plugins list through https requests

The devstack plugins list can be generated through web requests in
environments (such as the proposal slave) that lack copies of all
the relevant git repositories.

One downside to this is that there is no way of getting the last
modification time of the plugin.

Change-Id: I2c5c9282a8ad80014cad171a4dfbdc8f26044cd1
---
 data/devstack-plugins-registry.header   |  6 +--
 tools/generate-devstack-plugins-list.py | 59 +++++++++++++++++++++++++
 tools/generate-devstack-plugins-list.sh | 55 ++++++++++++++++-------
 3 files changed, 100 insertions(+), 20 deletions(-)
 create mode 100644 tools/generate-devstack-plugins-list.py

diff --git a/data/devstack-plugins-registry.header b/data/devstack-plugins-registry.header
index 9f8a99471c..687db0441c 100644
--- a/data/devstack-plugins-registry.header
+++ b/data/devstack-plugins-registry.header
@@ -14,6 +14,6 @@ The following are plugins that a script has found in the openstack/
 namespace, which includes but is not limited to official OpenStack
 projects.
 
-+------------------+------------------------------------------------------------+------------+
-|Plugin Name       |URL                                                         |Date        |
-+------------------+------------------------------------------------------------+------------+
++------------------+-------------------------------------------------------------------------+
+|Plugin Name       |URL                                                                      |
++------------------+-------------------------------------------------------------------------+
diff --git a/tools/generate-devstack-plugins-list.py b/tools/generate-devstack-plugins-list.py
new file mode 100644
index 0000000000..1fa550192e
--- /dev/null
+++ b/tools/generate-devstack-plugins-list.py
@@ -0,0 +1,59 @@
+#! /usr/bin/env python
+
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is intended to be run as part of a periodic proposal bot
+# job in OpenStack infrastructure.
+#
+# In order to function correctly, the environment in which the
+# script runs must have
+#   * network access to the review.openstack.org Gerrit API
+#     working directory
+#   * network access to https://git.openstack.org/cgit
+
+import json
+import requests
+
+url = 'https://review.openstack.org/projects/'
+
+# This is what a project looks like
+'''
+  "openstack-attic/akanda": {
+    "id": "openstack-attic%2Fakanda",
+    "state": "READ_ONLY"
+  },
+'''
+
+def is_in_openstack_namespace(proj):
+    return proj.startswith('openstack/')
+
+# Rather than returning a 404 for a nonexistent file, cgit delivers a
+# 0-byte response to a GET request.  It also does not provide a
+# Content-Length in a HEAD response, so the way we tell if a file exists
+# is to check the length of the entire GET response body.
+def has_devstack_plugin(proj):
+    r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/plugin.sh" % proj)
+    if len(r.text) > 0:
+        return True
+    else:
+        False
+
+r = requests.get(url)
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
+
+found_plugins = filter(has_devstack_plugin, projects)
+
+for project in found_plugins:
+    print project[10:]
diff --git a/tools/generate-devstack-plugins-list.sh b/tools/generate-devstack-plugins-list.sh
index 6e9e828978..01767854d7 100644
--- a/tools/generate-devstack-plugins-list.sh
+++ b/tools/generate-devstack-plugins-list.sh
@@ -19,11 +19,19 @@
 #
 # In order to function correctly, the environment in which the
 # script runs must have
+#   * a writable doc/source directory relative to the current
+#     working directory
+#   AND ( (
 #   * git
 #   * all git repos meant to be searched for plugins cloned and
 #     at the desired level of up-to-datedness
-#   * a writable doc/source directory relative to the current
+#   * the environment variable git_dir pointing to the location
+#   * of said git repositories
+#   ) OR (
+#   * network access to the review.openstack.org Gerrit API
 #     working directory
+#   * network access to https://git.openstack.org/cgit
+#   ))
 #
 # If a file named data/devstack-plugins-registry.header or
 # data/devstack-plugins-registry.footer is found relative to the
@@ -35,25 +43,38 @@ declare -A plugins
 
 test -r data/devstack-plugins-registry.header && cat data/devstack-plugins-registry.header
 
-pushd ${git_dir:-/opt/openstack} >/dev/null
-for i in *; do
-    pushd ${i} >/dev/null
-    if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
-        test -n "$output" && plugins[$i]=${output}
-    fi
+if test -n "$git_dir"; then
+    pushd ${git_dir} >/dev/null
+    for i in *; do
+        pushd ${i} >/dev/null
+        if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
+            test -n "$output" && plugins[$i]=${output}
+        fi
+        popd >/dev/null
+    done
     popd >/dev/null
-done
-popd >/dev/null
 
-sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
+    sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
 
-for k in "${sorted_plugins[@]}"; do
-    project=${k:0:18}
-    giturl="git://git.openstack.org/openstack/${k:0:26}"
-    pdate="${plugins[$k]}"
-    printf "|%-18s|%-60s|%-12s|\n" "${project}" "${giturl}" "${pdate}"
-    printf "+------------------+------------------------------------------------------------+------------+\n"
-done
+    for k in "${sorted_plugins[@]}"; do
+        project=${k:0:18}
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
+        pdate="${plugins[$k]}"
+        printf "|%-18s|%-60s (%-10s)|\n" "${project}" "${giturl}" "${pdate}"
+        printf "+------------------+-------------------------------------------------------------------------+\n"
+    done
+
+else
+    sorted_plugins=$(python tools/generate-devstack-plugins-list.py)
+
+    for k in ${sorted_plugins}; do
+        project=${k:0:18}
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
+        printf "|%-18s|%-73s|\n" "${project}" "${giturl}"
+        printf "+------------------+-------------------------------------------------------------------------+\n"
+    done
+
+fi
 
 test -r data/devstack-plugins-registry.footer && cat data/devstack-plugins-registry.footer
 ) > doc/source/plugin-registry.rst