Dynamically pull list of projects and versions

This also fixes this version check utility since it was utterly broken
before. While it did run to completion, it never properly returned any
versions that needed updating.

TrivialFix

Change-Id: If5d77f20fd8fc97679d58bc9019115157c5f5967

tools/version_check.py

@@ -33,35 +33,22 @@ if PROJECT_ROOT not in sys.path:
 
 from kolla.common import config as common_config
 
-
+# Filter list for non-projects
-# Use an OrderedDict to ensure the results are displayed alphabetically
+NOT_PROJECTS = [
-MAJOR_VERSIONS_MAP = collections.OrderedDict([
+    'nova-novncproxy',
-    ('ceilometer', 5),
+    'nova-spicehtml5proxy',
-    ('cinder', 7),
+    'openstack-base',
-    ('designate', 1),
+    'profiles'
-    ('glance', 11),
+]
-    ('gnocchi', 1),
-    ('heat', 5),
-    ('horizon', 8),
-    ('ironic', 4),
-    ('keystone', 8),
-    ('magnum', 1),
-    ('murano', 1),
-    ('neutron', 7),
-    ('nova', 12),
-    ('swift', 2),
-    ('zaqar', 1)
-])
-
 TARBALLS_BASE_URL = 'http://tarballs.openstack.org'
-VERSIONS = dict()
+VERSIONS = {'local': dict()}
 
 
 def retrieve_upstream_versions():
     upstream_versions = dict()
-    for project in MAJOR_VERSIONS_MAP:
+    for project in VERSIONS['local']:
         winner = None
-        series = MAJOR_VERSIONS_MAP[project]
+        series = VERSIONS['local'][project].split('.')[0]
         base = '{}/{}'.format(TARBALLS_BASE_URL, project)
         r = requests.get(base)
         s = bs(r.text, 'html.parser')
@@ -72,6 +59,9 @@ def retrieve_upstream_versions():
                     version.startswith('{}-{}'.format(project, series))):
                 split = '{}-|.tar.gz'.format(project)
                 candidate = re.split(split, version)[1]
+                # Ignore 2014, 2015 versions as they are older
+                if candidate.startswith('201'):
+                    continue
                 if not winner or more_recent(candidate, winner):
                     winner = candidate
 
@@ -79,26 +69,23 @@ def retrieve_upstream_versions():
             print('Could not find version for {}'.format(project))
             continue
 
+        if '-' in winner:
+            winner = winner.split('-')[1]
         upstream_versions[project] = winner
 
-    VERSIONS['upstream'] = upstream_versions
+    VERSIONS['upstream'] = collections.OrderedDict(
+        sorted(upstream_versions.items()))
 
 
-def retrieve_local_versions(conf):
+def retrieve_local_versions():
-    local_versions = dict()
+    for section in common_config.SOURCES:
-    for project in MAJOR_VERSIONS_MAP:
+        if section not in NOT_PROJECTS:
-        series = MAJOR_VERSIONS_MAP[project]
+            project = section.split('-')[0]
-        for project_section in [match.group(0) for match in
+            version = common_config.SOURCES[section]['location'].split(
-                                (re.search('^{}(?:-base)?$'.format(project),
+                '/')[-1].split('.tar.gz')[0]
-                                           section) for section in
+            if '-' in version:
-                                 conf._groups) if match]:
+                version = version.split('-')[1]
-            archive = conf[project_section]['location'].split('/')[-1]
+            VERSIONS['local'][project] = version
-            if (archive.endswith('.tar.gz') and
-                    archive.startswith('{}-{}'.format(project, series))):
-                split = '{}-|.tar.gz'.format(project)
-                local_versions[project] = re.split(split, archive)[1]
-
-    VERSIONS['local'] = local_versions
 
 
 def more_recent(candidate, reference):
@@ -130,8 +117,8 @@ def main():
     conf = cfg.ConfigOpts()
     common_config.parse(conf, sys.argv[1:], prog='kolla-build')
 
+    retrieve_local_versions()
     retrieve_upstream_versions()
-    retrieve_local_versions(conf)
 
     compare_versions()