Adjust multipip usage.

Use json as the output format and only use stderr for
when the multipip program really fails.

Instead of outputting to the screen all incompatible errors
just output the incompatible versions to the output json
data structure and use this in the main anvil program.

Use defaultdict + list in multipip where appropriate.

Ensure that as documented we select the first matching requirement
that comes out of multipip instead of the last (when incompat. matches
happen).

Adjust documentation for new json output format.

Change-Id: I6403182066ef01cb3f7f26c305ab5577a00043ae

anvil/packaging/base.py

@@ -18,9 +18,11 @@
 # R0921: Abstract class not referenced
 #pylint: disable=R0902,R0921
 
+import json
 import pkg_resources
 
 from anvil import colorizer
+from anvil import exceptions as excp
 from anvil import log as logging
 from anvil import shell as sh
 from anvil import utils
@@ -149,7 +151,7 @@ class DependencyHandler(object):
                     req = pkg_resources.Requirement.parse(line)
                     new_lines.append(str(forced_by_key[req.key]))
                 except:
-                    # we don't force the package or it has a bad format
+                    # We don't force the package or it has a bad format
                     new_lines.append(line)
             contents = "# Cleaned on %s\n\n%s\n" % (
                 utils.iso8601(), "\n".join(new_lines))
@@ -171,34 +173,46 @@ class DependencyHandler(object):
         ]
         cmdline = cmdline + extra_pips + ["-r"] + requires_files
 
-        output = sh.execute(cmdline, check_exit_code=False)
+        output = sh.execute(cmdline)
-        conflict_descr = output[1].strip()
+        contents = json.loads(output[0])
+
+        # Figure out which ones were easily matched.
+        self.pips_to_install = []
+        for pkg in contents.get('compatibles', []):
+            if pkg.lower() not in OPENSTACK_PACKAGES:
+                self.pips_to_install.append(pkg)
+
+        # Figure out which ones we are forced to install.
         forced_keys = set()
-        if conflict_descr:
+        for (k, req_list) in contents.get('incompatibles', {}):
-            for line in conflict_descr.splitlines():
+            forced_keys.add(k.lower())
-                LOG.warning(line)
+            # Select the first.
-                if line.endswith(": incompatible requirements"):
+            if req_list and req_list[0] not in self.pips_to_install:
-                    forced_keys.add(line.split(":", 1)[0].lower())
+                self.pips_to_install.append(req_list[0])
-        self.pips_to_install = [
+
-            pkg
-            for pkg in utils.splitlines_not_empty(output[0])
-            if pkg.lower() not in OPENSTACK_PACKAGES]
-        sh.write_file(self.gathered_requires_filename,
-                      "\n".join(self.pips_to_install))
         if not self.pips_to_install:
-            LOG.error("No dependencies for OpenStack found."
+            LOG.error("No dependencies for OpenStack found. "
                       "Something went wrong. Please check:")
             LOG.error("'%s'" % "' '".join(cmdline))
-            raise RuntimeError("No dependencies for OpenStack found")
+            raise excp.AnvilException("No dependencies for OpenStack found!")
 
         utils.log_iterable(sorted(self.pips_to_install),
                            logger=LOG,
-                           header="Full known Python dependency list")
+                           header="Python dependency list (all)")
+        sh.write_file(self.gathered_requires_filename,
+                      "\n".join(self.pips_to_install))
+
         self.forced_packages = []
-        for pip in self.pips_to_install:
+        for k in forced_keys:
-            req = pkg_resources.Requirement.parse(pip)
+            for pip in self.pips_to_install:
-            if req.key in forced_keys:
+                req = pkg_resources.Requirement.parse(pip)
-                self.forced_packages.append(req)
+                if req.key == k:
+                    self.forced_packages.append(req)
+                    break
+        if self.forced_packages:
+            utils.log_iterable(sorted(self.forced_packages),
+                               logger=LOG,
+                               header="Python dependency list (forced)")
         sh.write_file(self.forced_requires_filename,
                       "\n".join(str(req) for req in self.forced_packages))
 

tools/README.rst

@@ -12,27 +12,23 @@ multipip
 Use `multipip` to join these requirements::
     
     $ multipip 'nose>=1.2' 'nose>=2' 'nose<4'
-    nose>=2,<4
+    {"compatibles": ["nose>=2,<4"], "incompatibles": {}}
 
 Files of requirements can be used as well::
 
     $ cat pip-requires 
     nose<4
     $ multipip 'nose>=1.2' 'nose>=2' -r pip-requires 
-    nose>=2,<4
+    {"compatibles": ["nose>=2,<4"], "incompatibles": {}}
 
-`multipip` prints error messages for incompatible requirements to
+`multipip` prints error messages for badly formated requirements and exits early
-stderr and chooses the first one::
+and for incompatible requirements provides you which package was incompatible
+and which versions were found to be problematic::
 
     $ cat pip-requires 
     pip==1.3
     $ multipip 'pip==1.2' -r pip-requires 
-    pip: incompatible requirements
+    {"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}}
-    Choosing:
-    	command line: pip==1.2
-    Conflicting:
-    	-r pip-requires (line 1): pip==1.3
-    pip==1.2
     
 It is possible to filter some packages from printed output. This can
 be useful for a huge `pip-requires` file::
@@ -42,7 +38,7 @@ be useful for a huge `pip-requires` file::
     pip==1.2
     nose>=1.2
     $ multipip -r pip-requires --ignore-packages nose
-    pip==1.2
+    {"compatibles": ["pip==1.2"], "incompatibles": {}}
 
 Installed packages can be filtered, too (they are taken from `pip
 freeze`)::
@@ -54,8 +50,8 @@ freeze`)::
     $ pip freeze | grep nose
     nose==1.1.2
     $ multipip -r pip-requires --ignore-installed
-    pip==1.2
+    {"compatibles": ["pip==1.2"], "incompatibles": {}}
-    
+
 py2rpm
 ------
 

tools/multipip

@@ -1,7 +1,9 @@
 #!/usr/bin/python
 
 import argparse
+import collections
 import distutils.spawn
+import json
 import logging
 import os
 import subprocess
@@ -14,7 +16,6 @@ import pkg_resources
 
 
 BAD_REQUIREMENTS = 2
-INCOMPATIBLE_REQUIREMENTS = 3
 logger = logging.getLogger()
 
 
@@ -84,12 +85,12 @@ def setup_logging(options):
     logger.setLevel(level)
 
 
-incompatibles = set()
+incompatibles = collections.defaultdict(list)
 joined_requirements = []
 
 
 def install_requirement_ensure_req_field(req):
-    if not req.req:
+    if not hasattr(req, 'req') or not req.req:
         # pip 0.8 or so
         link = pip.index.Link(req.url)
         name = link.egg_fragment
@@ -117,18 +118,15 @@ def install_requirement_parse(line, comes_from):
     return install_requirement_ensure_req_field(req)
 
 
-def incompatible_requirement(chosen, conflicting):
+def add_incompatible_requirement(chosen, conflicting):
-    if chosen.req.key not in incompatibles:
+    global incompatibles
-        incompatibles.add(chosen.req.key)
+
-        print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key
+    key = chosen.req.key
-        print >> sys.stderr, "Choosing:"
+    if key not in incompatibles:
-        print >> sys.stderr, ("\t%s: %s" %
+        incompatibles[key].extend([install_requirement_str(conflicting),
-                              (chosen.comes_from,
+                                   install_requirement_str(chosen)])
-                               install_requirement_str(chosen)))
+    else:
-        print >> sys.stderr, "Conflicting:"
+        incompatibles[key].append(install_requirement_str(chosen))
-    print >> sys.stderr, ("\t%s: %s" %
-                          (conflicting.comes_from,
-                           install_requirement_str(conflicting)))
 
 
 def parse_requirements(options):
@@ -136,22 +134,24 @@ def parse_requirements(options):
 
     :return: tuple (all, ignored) of InstallRequirement
     """
-    all_requirements = {}
+    all_requirements = collections.defaultdict(list)
-    for req_spec in options.requirement_specs:
+    # CLI directly provided requirements first.
-        try:
-            req = install_requirement_parse(req_spec, "command line")
-            all_requirements.setdefault(req.req.key, []).append(req)
-        except Exception as ex:
-            logger.error("Cannot parse `%s': %s" % (req_spec, ex))
-            sys.exit(BAD_REQUIREMENTS)
     for filename in options.requirements:
         try:
             for req in pip.req.parse_requirements(filename, options=options):
                 req = install_requirement_ensure_req_field(req)
-                all_requirements.setdefault(req.req.key, []).append(req)
+                all_requirements[req.req.key].append(req)
         except Exception as ex:
             logger.error("Cannot parse `%s': %s" % (filename, ex))
             sys.exit(BAD_REQUIREMENTS)
+    # CLI provided files after that.
+    for req_spec in options.requirement_specs:
+        try:
+            req = install_requirement_parse(req_spec, "command line")
+            all_requirements[req.req.key].append(req)
+        except Exception as ex:
+            logger.error("Cannot parse `%s': %s" % (req_spec, ex))
+            sys.exit(BAD_REQUIREMENTS)
     ignored_requirements = []
     for req_spec in options.ignore_packages:
         try:
@@ -249,6 +249,7 @@ def join_one_requirement(req_list):
 
 def join_requirements(options):
     global joined_requirements
+
     all_requirements, ignored_requirements = parse_requirements(options)
     skip_keys = set(pkg.req.key for pkg in ignored_requirements)
     installed_by_key = {}
@@ -275,7 +276,7 @@ def join_requirements(options):
                     "%s>=%s" % (installed_req.project_name,
                                 installed_req.specs[0][1]),
                     "pip freeze")
-                incompatible_requirement(frozen_req, joined_req)
+                add_incompatible_requirement(frozen_req, joined_req)
                 joined_req = frozen_req
         joined_requirements.append(joined_req.req)
 
@@ -297,26 +298,36 @@ def join_requirements(options):
         if exact_version:
             for req in req_list:
                 if not exact_version in req.req:
-                    incompatible_requirement(joined_req, req)
+                    add_incompatible_requirement(joined_req, req)
         else:
             for req in req_list:
                 for parsed, trans, op, ver in req.req.index:
                     if op[0] == "=":
                         if parsed in conflicts:
-                            incompatible_requirement(joined_req, req)
+                            add_incompatible_requirement(joined_req, req)
                             break
                     elif not segment_ok and op[0] == "<":
                         # analyse lower bound: x >= A or x > A
                         if (lower_version > parsed or (
                                 lower_version == parsed and
                                 (lower_strict or len(op) != 2))):
-                            incompatible_requirement(joined_req, req)
+                            add_incompatible_requirement(joined_req, req)
                             break
 
 
 def print_requirements():
-    for req in sorted(joined_requirements, key=lambda x: x.key):
+    global joined_requirements
-        print req
+    global incompatibles
+
+    requirements = {
+        'compatibles': [],
+        'incompatibles': {},
+    }
+    for r in joined_requirements:
+        if r.key not in incompatibles:
+            requirements['compatibles'].append(str(r))
+    requirements['incompatibles'].update(incompatibles)
+    print(json.dumps(requirements))
 
 
 def main():
@@ -325,8 +336,6 @@ def main():
     setup_logging(options)
     join_requirements(options)
     print_requirements()
-    if incompatibles:
-        sys.exit(INCOMPATIBLE_REQUIREMENTS)
 
 
 if __name__ == "__main__":