From 86bf535a0a0c484719d1c552a905958cb75683cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 4 Jun 2013 15:25:36 -0700 Subject: [PATCH] Adjust multipip usage. Use json as the output format and only use stderr for when the multipip program really fails. Instead of outputting to the screen all incompatible errors just output the incompatible versions to the output json data structure and use this in the main anvil program. Use defaultdict + list in multipip where appropriate. Ensure that as documented we select the first matching requirement that comes out of multipip instead of the last (when incompat. matches happen). Adjust documentation for new json output format. Change-Id: I6403182066ef01cb3f7f26c305ab5577a00043ae --- anvil/packaging/base.py | 56 +++++++++++++++++++------------ tools/README.rst | 22 +++++-------- tools/multipip | 73 +++++++++++++++++++++++------------------ 3 files changed, 85 insertions(+), 66 deletions(-) diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 4cab2361..81b3b37c 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -18,9 +18,11 @@ # R0921: Abstract class not referenced #pylint: disable=R0902,R0921 +import json import pkg_resources from anvil import colorizer +from anvil import exceptions as excp from anvil import log as logging from anvil import shell as sh from anvil import utils @@ -149,7 +151,7 @@ class DependencyHandler(object): req = pkg_resources.Requirement.parse(line) new_lines.append(str(forced_by_key[req.key])) except: - # we don't force the package or it has a bad format + # We don't force the package or it has a bad format new_lines.append(line) contents = "# Cleaned on %s\n\n%s\n" % ( utils.iso8601(), "\n".join(new_lines)) @@ -171,34 +173,46 @@ class DependencyHandler(object): ] cmdline = cmdline + extra_pips + ["-r"] + requires_files - output = sh.execute(cmdline, check_exit_code=False) - conflict_descr = output[1].strip() + output = sh.execute(cmdline) + contents = json.loads(output[0]) + + # Figure out which ones were easily matched. + self.pips_to_install = [] + for pkg in contents.get('compatibles', []): + if pkg.lower() not in OPENSTACK_PACKAGES: + self.pips_to_install.append(pkg) + + # Figure out which ones we are forced to install. forced_keys = set() - if conflict_descr: - for line in conflict_descr.splitlines(): - LOG.warning(line) - if line.endswith(": incompatible requirements"): - forced_keys.add(line.split(":", 1)[0].lower()) - self.pips_to_install = [ - pkg - for pkg in utils.splitlines_not_empty(output[0]) - if pkg.lower() not in OPENSTACK_PACKAGES] - sh.write_file(self.gathered_requires_filename, - "\n".join(self.pips_to_install)) + for (k, req_list) in contents.get('incompatibles', {}): + forced_keys.add(k.lower()) + # Select the first. + if req_list and req_list[0] not in self.pips_to_install: + self.pips_to_install.append(req_list[0]) + if not self.pips_to_install: - LOG.error("No dependencies for OpenStack found." + LOG.error("No dependencies for OpenStack found. " "Something went wrong. Please check:") LOG.error("'%s'" % "' '".join(cmdline)) - raise RuntimeError("No dependencies for OpenStack found") + raise excp.AnvilException("No dependencies for OpenStack found!") utils.log_iterable(sorted(self.pips_to_install), logger=LOG, - header="Full known Python dependency list") + header="Python dependency list (all)") + sh.write_file(self.gathered_requires_filename, + "\n".join(self.pips_to_install)) + self.forced_packages = [] - for pip in self.pips_to_install: - req = pkg_resources.Requirement.parse(pip) - if req.key in forced_keys: - self.forced_packages.append(req) + for k in forced_keys: + for pip in self.pips_to_install: + req = pkg_resources.Requirement.parse(pip) + if req.key == k: + self.forced_packages.append(req) + break + if self.forced_packages: + utils.log_iterable(sorted(self.forced_packages), + logger=LOG, + header="Python dependency list (forced)") sh.write_file(self.forced_requires_filename, "\n".join(str(req) for req in self.forced_packages)) diff --git a/tools/README.rst b/tools/README.rst index d1a7106c..647e7296 100644 --- a/tools/README.rst +++ b/tools/README.rst @@ -12,27 +12,23 @@ multipip Use `multipip` to join these requirements:: $ multipip 'nose>=1.2' 'nose>=2' 'nose<4' - nose>=2,<4 + {"compatibles": ["nose>=2,<4"], "incompatibles": {}} Files of requirements can be used as well:: $ cat pip-requires nose<4 $ multipip 'nose>=1.2' 'nose>=2' -r pip-requires - nose>=2,<4 + {"compatibles": ["nose>=2,<4"], "incompatibles": {}} -`multipip` prints error messages for incompatible requirements to -stderr and chooses the first one:: +`multipip` prints error messages for badly formated requirements and exits early +and for incompatible requirements provides you which package was incompatible +and which versions were found to be problematic:: $ cat pip-requires pip==1.3 $ multipip 'pip==1.2' -r pip-requires - pip: incompatible requirements - Choosing: - command line: pip==1.2 - Conflicting: - -r pip-requires (line 1): pip==1.3 - pip==1.2 + {"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}} It is possible to filter some packages from printed output. This can be useful for a huge `pip-requires` file:: @@ -42,7 +38,7 @@ be useful for a huge `pip-requires` file:: pip==1.2 nose>=1.2 $ multipip -r pip-requires --ignore-packages nose - pip==1.2 + {"compatibles": ["pip==1.2"], "incompatibles": {}} Installed packages can be filtered, too (they are taken from `pip freeze`):: @@ -54,8 +50,8 @@ freeze`):: $ pip freeze | grep nose nose==1.1.2 $ multipip -r pip-requires --ignore-installed - pip==1.2 - + {"compatibles": ["pip==1.2"], "incompatibles": {}} + py2rpm ------ diff --git a/tools/multipip b/tools/multipip index 2c6b3a05..7042bd17 100755 --- a/tools/multipip +++ b/tools/multipip @@ -1,7 +1,9 @@ #!/usr/bin/python import argparse +import collections import distutils.spawn +import json import logging import os import subprocess @@ -14,7 +16,6 @@ import pkg_resources BAD_REQUIREMENTS = 2 -INCOMPATIBLE_REQUIREMENTS = 3 logger = logging.getLogger() @@ -84,12 +85,12 @@ def setup_logging(options): logger.setLevel(level) -incompatibles = set() +incompatibles = collections.defaultdict(list) joined_requirements = [] def install_requirement_ensure_req_field(req): - if not req.req: + if not hasattr(req, 'req') or not req.req: # pip 0.8 or so link = pip.index.Link(req.url) name = link.egg_fragment @@ -117,18 +118,15 @@ def install_requirement_parse(line, comes_from): return install_requirement_ensure_req_field(req) -def incompatible_requirement(chosen, conflicting): - if chosen.req.key not in incompatibles: - incompatibles.add(chosen.req.key) - print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key - print >> sys.stderr, "Choosing:" - print >> sys.stderr, ("\t%s: %s" % - (chosen.comes_from, - install_requirement_str(chosen))) - print >> sys.stderr, "Conflicting:" - print >> sys.stderr, ("\t%s: %s" % - (conflicting.comes_from, - install_requirement_str(conflicting))) +def add_incompatible_requirement(chosen, conflicting): + global incompatibles + + key = chosen.req.key + if key not in incompatibles: + incompatibles[key].extend([install_requirement_str(conflicting), + install_requirement_str(chosen)]) + else: + incompatibles[key].append(install_requirement_str(chosen)) def parse_requirements(options): @@ -136,22 +134,24 @@ def parse_requirements(options): :return: tuple (all, ignored) of InstallRequirement """ - all_requirements = {} - for req_spec in options.requirement_specs: - try: - req = install_requirement_parse(req_spec, "command line") - all_requirements.setdefault(req.req.key, []).append(req) - except Exception as ex: - logger.error("Cannot parse `%s': %s" % (req_spec, ex)) - sys.exit(BAD_REQUIREMENTS) + all_requirements = collections.defaultdict(list) + # CLI directly provided requirements first. for filename in options.requirements: try: for req in pip.req.parse_requirements(filename, options=options): req = install_requirement_ensure_req_field(req) - all_requirements.setdefault(req.req.key, []).append(req) + all_requirements[req.req.key].append(req) except Exception as ex: logger.error("Cannot parse `%s': %s" % (filename, ex)) sys.exit(BAD_REQUIREMENTS) + # CLI provided files after that. + for req_spec in options.requirement_specs: + try: + req = install_requirement_parse(req_spec, "command line") + all_requirements[req.req.key].append(req) + except Exception as ex: + logger.error("Cannot parse `%s': %s" % (req_spec, ex)) + sys.exit(BAD_REQUIREMENTS) ignored_requirements = [] for req_spec in options.ignore_packages: try: @@ -249,6 +249,7 @@ def join_one_requirement(req_list): def join_requirements(options): global joined_requirements + all_requirements, ignored_requirements = parse_requirements(options) skip_keys = set(pkg.req.key for pkg in ignored_requirements) installed_by_key = {} @@ -275,7 +276,7 @@ def join_requirements(options): "%s>=%s" % (installed_req.project_name, installed_req.specs[0][1]), "pip freeze") - incompatible_requirement(frozen_req, joined_req) + add_incompatible_requirement(frozen_req, joined_req) joined_req = frozen_req joined_requirements.append(joined_req.req) @@ -297,26 +298,36 @@ def join_requirements(options): if exact_version: for req in req_list: if not exact_version in req.req: - incompatible_requirement(joined_req, req) + add_incompatible_requirement(joined_req, req) else: for req in req_list: for parsed, trans, op, ver in req.req.index: if op[0] == "=": if parsed in conflicts: - incompatible_requirement(joined_req, req) + add_incompatible_requirement(joined_req, req) break elif not segment_ok and op[0] == "<": # analyse lower bound: x >= A or x > A if (lower_version > parsed or ( lower_version == parsed and (lower_strict or len(op) != 2))): - incompatible_requirement(joined_req, req) + add_incompatible_requirement(joined_req, req) break def print_requirements(): - for req in sorted(joined_requirements, key=lambda x: x.key): - print req + global joined_requirements + global incompatibles + + requirements = { + 'compatibles': [], + 'incompatibles': {}, + } + for r in joined_requirements: + if r.key not in incompatibles: + requirements['compatibles'].append(str(r)) + requirements['incompatibles'].update(incompatibles) + print(json.dumps(requirements)) def main(): @@ -325,8 +336,6 @@ def main(): setup_logging(options) join_requirements(options) print_requirements() - if incompatibles: - sys.exit(INCOMPATIBLE_REQUIREMENTS) if __name__ == "__main__":