Improve script to check used source versions

* rename version_check.py to version-check.py
* move requirement beautifulsoup4 from requirements to test-requirements
* add new requirement PrettyTable to test-requirements
* add debug logging messages (can be enabled with --debug parameter)
* add support to load configured versions from a kolla-build.conf
  file (can be used with --config-file parameter)
* use PrettyTable to print out a nice result table

---snip---
+---------+-----------------+----------------+------------------...
| Project | Current version | Latest version | Comparing changes...
+---------+-----------------+----------------+------------------...
| aodh    | 2.0.3           | 2.0.4          | https://github.co...
| cinder  | 8.0.0           | 8.1.0          | https://github.co...
| ironic  | 5.1.1           | 5.1.2          | https://github.co...
| murano  | 2.0.0           | 2.0.1          | https://github.co...
| nova    | 13.1.0          | 13.1.1         | https://github.co...
+---------+-----------------+----------------+------------------...
---snap---

TrivialFix

Change-Id: I5b5d3db34396b2fc95714273612563ea14e78cf8

requirements.txt

@@ -9,6 +9,5 @@ GitPython>=1.0.1 # BSD License (3 clause)
 six>=1.9.0 # MIT
 oslo.config>=3.14.0 # Apache-2.0
 graphviz>=0.4.0 # MIT License
-beautifulsoup4 # MIT
 setuptools!=24.0.0,>=16.0 # PSF/ZPL
 pycrypto>=2.6 # Public Domain

test-requirements.txt

@@ -3,12 +3,14 @@
 # process, which may cause wedges in the gate later.
 bandit>=1.0.1 # Apache-2.0
 bashate>=0.2 # Apache-2.0
+beautifulsoup4  # MIT
 doc8 # Apache-2.0
 hacking>=0.10.0
 oslo.log>=1.14.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
 reno>=1.8.0 # Apache2
+PrettyTable>=0.7,<0.8  # BSD
 PyYAML>=3.1.0 # MIT
 python-barbicanclient>=4.0.0 # Apache-2.0
 python-ceilometerclient>=2.5.0 # Apache-2.0

tools/version-check.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import collections
+import logging
 import os
 import re
 import sys
@@ -20,6 +21,7 @@ import sys
 from bs4 import BeautifulSoup as bs
 from oslo_config import cfg
 import pkg_resources
+from prettytable import PrettyTable
 import requests
 
 PROJECT_ROOT = os.path.abspath(os.path.join(
@@ -33,6 +35,9 @@ if PROJECT_ROOT not in sys.path:
 
 from kolla.common import config as common_config
 
+logging.basicConfig(format="%(message)s")
+LOG = logging.getLogger('version-check')
+
 # Filter list for non-projects
 NOT_PROJECTS = [
     'nova-novncproxy',
@@ -50,6 +55,8 @@ def retrieve_upstream_versions():
         winner = None
         series = VERSIONS['local'][project].split('.')[0]
         base = '{}/{}'.format(TARBALLS_BASE_URL, project)
+        LOG.debug("Getting latest version for project %s from %s",
+                  project, base)
         r = requests.get(base)
         s = bs(r.text, 'html.parser')
 
@@ -66,25 +73,37 @@ def retrieve_upstream_versions():
                     winner = candidate
 
         if not winner:
-            print('Could not find version for {}'.format(project))
+            LOG.warning("Could not find a version for %s", project)
             continue
 
         if '-' in winner:
             winner = winner.split('-')[1]
         upstream_versions[project] = winner
+        LOG.debug("Found latest version %s for project %s", winner, project)
 
     VERSIONS['upstream'] = collections.OrderedDict(
         sorted(upstream_versions.items()))
 
 
-def retrieve_local_versions():
+def retrieve_local_versions(conf):
     for section in common_config.SOURCES:
-        if section not in NOT_PROJECTS:
+        if section in NOT_PROJECTS:
+            continue
+
         project = section.split('-')[0]
-            version = common_config.SOURCES[section]['location'].split(
+
-                '/')[-1].split('.tar.gz')[0]
+        if section not in conf.list_all_sections():
+            LOG.debug("Project %s not found in configuration file, using "
+                      "default from kolla.common.config", project)
+            raw_version = common_config.SOURCES[section]['location']
+        else:
+            raw_version = getattr(conf, section).location
+
+        version = raw_version.split('/')[-1].split('.tar.gz')[0]
         if '-' in version:
             version = version.split('-')[1]
+
+        LOG.debug("Use local version %s for project %s", version, project)
         VERSIONS['local'][project] = version
 
 
@@ -100,24 +119,40 @@ def diff_link(project, old_ref, new_ref):
 
 def compare_versions():
     up_to_date = True
+    result = PrettyTable(["Project", "Current version",
+                          "Latest version", "Comparing changes"])
+    result.align = "l"
+
     for project in VERSIONS['upstream']:
-        if project in VERSIONS['local']:
+        if project not in VERSIONS['local']:
+            continue
+
         upstream_version = VERSIONS['upstream'][project]
         local_version = VERSIONS['local'][project]
+
         if more_recent(upstream_version, local_version):
-                print("{} has newer version {} > {}, see diff at {}".format(
+            result.add_row([
-                    project, upstream_version, local_version,
+                project,
-                    diff_link(project, local_version, upstream_version)))
+                VERSIONS['local'][project],
+                VERSIONS['upstream'][project],
+                diff_link(project, local_version, upstream_version)
+            ])
             up_to_date = False
+
     if up_to_date:
-        print("Everything is up to date")
+        result = "Everything is up to date"
+
+    print(result)
 
 
 def main():
     conf = cfg.ConfigOpts()
-    common_config.parse(conf, sys.argv[1:], prog='kolla-build')
+    common_config.parse(conf, sys.argv[1:], prog='version-check')
 
-    retrieve_local_versions()
+    if conf.debug:
+        LOG.setLevel(logging.DEBUG)
+
+    retrieve_local_versions(conf)
     retrieve_upstream_versions()
 
     compare_versions()