Adjust multipip usage.
Use json as the output format and only use stderr for when the multipip program really fails. Instead of outputting to the screen all incompatible errors just output the incompatible versions to the output json data structure and use this in the main anvil program. Use defaultdict + list in multipip where appropriate. Ensure that as documented we select the first matching requirement that comes out of multipip instead of the last (when incompat. matches happen). Adjust documentation for new json output format. Change-Id: I6403182066ef01cb3f7f26c305ab5577a00043ae
This commit is contained in:
Joshua Harlow
parent
aa6d4db7c4
commit
86bf535a0a
@ -18,9 +18,11 @@
|
||||
# R0921: Abstract class not referenced
|
||||
#pylint: disable=R0902,R0921
|
||||
|
||||
import json
|
||||
import pkg_resources
|
||||
|
||||
from anvil import colorizer
|
||||
from anvil import exceptions as excp
|
||||
from anvil import log as logging
|
||||
from anvil import shell as sh
|
||||
from anvil import utils
|
||||
@ -149,7 +151,7 @@ class DependencyHandler(object):
|
||||
req = pkg_resources.Requirement.parse(line)
|
||||
new_lines.append(str(forced_by_key[req.key]))
|
||||
except:
|
||||
# we don't force the package or it has a bad format
|
||||
# We don't force the package or it has a bad format
|
||||
new_lines.append(line)
|
||||
contents = "# Cleaned on %s\n\n%s\n" % (
|
||||
utils.iso8601(), "\n".join(new_lines))
|
||||
@ -171,34 +173,46 @@ class DependencyHandler(object):
|
||||
]
|
||||
cmdline = cmdline + extra_pips + ["-r"] + requires_files
|
||||
|
||||
output = sh.execute(cmdline, check_exit_code=False)
|
||||
conflict_descr = output[1].strip()
|
||||
output = sh.execute(cmdline)
|
||||
contents = json.loads(output[0])
|
||||
|
||||
# Figure out which ones were easily matched.
|
||||
self.pips_to_install = []
|
||||
for pkg in contents.get('compatibles', []):
|
||||
if pkg.lower() not in OPENSTACK_PACKAGES:
|
||||
self.pips_to_install.append(pkg)
|
||||
|
||||
# Figure out which ones we are forced to install.
|
||||
forced_keys = set()
|
||||
if conflict_descr:
|
||||
for line in conflict_descr.splitlines():
|
||||
LOG.warning(line)
|
||||
if line.endswith(": incompatible requirements"):
|
||||
forced_keys.add(line.split(":", 1)[0].lower())
|
||||
self.pips_to_install = [
|
||||
pkg
|
||||
for pkg in utils.splitlines_not_empty(output[0])
|
||||
if pkg.lower() not in OPENSTACK_PACKAGES]
|
||||
sh.write_file(self.gathered_requires_filename,
|
||||
"\n".join(self.pips_to_install))
|
||||
for (k, req_list) in contents.get('incompatibles', {}):
|
||||
forced_keys.add(k.lower())
|
||||
# Select the first.
|
||||
if req_list and req_list[0] not in self.pips_to_install:
|
||||
self.pips_to_install.append(req_list[0])
|
||||
|
||||
if not self.pips_to_install:
|
||||
LOG.error("No dependencies for OpenStack found."
|
||||
LOG.error("No dependencies for OpenStack found. "
|
||||
"Something went wrong. Please check:")
|
||||
LOG.error("'%s'" % "' '".join(cmdline))
|
||||
raise RuntimeError("No dependencies for OpenStack found")
|
||||
raise excp.AnvilException("No dependencies for OpenStack found!")
|
||||
|
||||
utils.log_iterable(sorted(self.pips_to_install),
|
||||
logger=LOG,
|
||||
header="Full known Python dependency list")
|
||||
header="Python dependency list (all)")
|
||||
sh.write_file(self.gathered_requires_filename,
|
||||
"\n".join(self.pips_to_install))
|
||||
|
||||
self.forced_packages = []
|
||||
for pip in self.pips_to_install:
|
||||
req = pkg_resources.Requirement.parse(pip)
|
||||
if req.key in forced_keys:
|
||||
self.forced_packages.append(req)
|
||||
for k in forced_keys:
|
||||
for pip in self.pips_to_install:
|
||||
req = pkg_resources.Requirement.parse(pip)
|
||||
if req.key == k:
|
||||
self.forced_packages.append(req)
|
||||
break
|
||||
if self.forced_packages:
|
||||
utils.log_iterable(sorted(self.forced_packages),
|
||||
logger=LOG,
|
||||
header="Python dependency list (forced)")
|
||||
sh.write_file(self.forced_requires_filename,
|
||||
"\n".join(str(req) for req in self.forced_packages))
|
||||
|
||||
|
||||
@ -12,27 +12,23 @@ multipip
|
||||
Use `multipip` to join these requirements::
|
||||
|
||||
$ multipip 'nose>=1.2' 'nose>=2' 'nose<4'
|
||||
nose>=2,<4
|
||||
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
|
||||
|
||||
Files of requirements can be used as well::
|
||||
|
||||
$ cat pip-requires
|
||||
nose<4
|
||||
$ multipip 'nose>=1.2' 'nose>=2' -r pip-requires
|
||||
nose>=2,<4
|
||||
{"compatibles": ["nose>=2,<4"], "incompatibles": {}}
|
||||
|
||||
`multipip` prints error messages for incompatible requirements to
|
||||
stderr and chooses the first one::
|
||||
`multipip` prints error messages for badly formated requirements and exits early
|
||||
and for incompatible requirements provides you which package was incompatible
|
||||
and which versions were found to be problematic::
|
||||
|
||||
$ cat pip-requires
|
||||
pip==1.3
|
||||
$ multipip 'pip==1.2' -r pip-requires
|
||||
pip: incompatible requirements
|
||||
Choosing:
|
||||
command line: pip==1.2
|
||||
Conflicting:
|
||||
-r pip-requires (line 1): pip==1.3
|
||||
pip==1.2
|
||||
{"compatibles": [], "incompatibles": {"pip": ["pip==1.2", "pip==1.3"]}}
|
||||
|
||||
It is possible to filter some packages from printed output. This can
|
||||
be useful for a huge `pip-requires` file::
|
||||
@ -42,7 +38,7 @@ be useful for a huge `pip-requires` file::
|
||||
pip==1.2
|
||||
nose>=1.2
|
||||
$ multipip -r pip-requires --ignore-packages nose
|
||||
pip==1.2
|
||||
{"compatibles": ["pip==1.2"], "incompatibles": {}}
|
||||
|
||||
Installed packages can be filtered, too (they are taken from `pip
|
||||
freeze`)::
|
||||
@ -54,7 +50,7 @@ freeze`)::
|
||||
$ pip freeze | grep nose
|
||||
nose==1.1.2
|
||||
$ multipip -r pip-requires --ignore-installed
|
||||
pip==1.2
|
||||
{"compatibles": ["pip==1.2"], "incompatibles": {}}
|
||||
|
||||
py2rpm
|
||||
------
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import distutils.spawn
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
@ -14,7 +16,6 @@ import pkg_resources
|
||||
|
||||
|
||||
BAD_REQUIREMENTS = 2
|
||||
INCOMPATIBLE_REQUIREMENTS = 3
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
@ -84,12 +85,12 @@ def setup_logging(options):
|
||||
logger.setLevel(level)
|
||||
|
||||
|
||||
incompatibles = set()
|
||||
incompatibles = collections.defaultdict(list)
|
||||
joined_requirements = []
|
||||
|
||||
|
||||
def install_requirement_ensure_req_field(req):
|
||||
if not req.req:
|
||||
if not hasattr(req, 'req') or not req.req:
|
||||
# pip 0.8 or so
|
||||
link = pip.index.Link(req.url)
|
||||
name = link.egg_fragment
|
||||
@ -117,18 +118,15 @@ def install_requirement_parse(line, comes_from):
|
||||
return install_requirement_ensure_req_field(req)
|
||||
|
||||
|
||||
def incompatible_requirement(chosen, conflicting):
|
||||
if chosen.req.key not in incompatibles:
|
||||
incompatibles.add(chosen.req.key)
|
||||
print >> sys.stderr, "%s: incompatible requirements" % chosen.req.key
|
||||
print >> sys.stderr, "Choosing:"
|
||||
print >> sys.stderr, ("\t%s: %s" %
|
||||
(chosen.comes_from,
|
||||
install_requirement_str(chosen)))
|
||||
print >> sys.stderr, "Conflicting:"
|
||||
print >> sys.stderr, ("\t%s: %s" %
|
||||
(conflicting.comes_from,
|
||||
install_requirement_str(conflicting)))
|
||||
def add_incompatible_requirement(chosen, conflicting):
|
||||
global incompatibles
|
||||
|
||||
key = chosen.req.key
|
||||
if key not in incompatibles:
|
||||
incompatibles[key].extend([install_requirement_str(conflicting),
|
||||
install_requirement_str(chosen)])
|
||||
else:
|
||||
incompatibles[key].append(install_requirement_str(chosen))
|
||||
|
||||
|
||||
def parse_requirements(options):
|
||||
@ -136,22 +134,24 @@ def parse_requirements(options):
|
||||
|
||||
:return: tuple (all, ignored) of InstallRequirement
|
||||
"""
|
||||
all_requirements = {}
|
||||
for req_spec in options.requirement_specs:
|
||||
try:
|
||||
req = install_requirement_parse(req_spec, "command line")
|
||||
all_requirements.setdefault(req.req.key, []).append(req)
|
||||
except Exception as ex:
|
||||
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
|
||||
sys.exit(BAD_REQUIREMENTS)
|
||||
all_requirements = collections.defaultdict(list)
|
||||
# CLI directly provided requirements first.
|
||||
for filename in options.requirements:
|
||||
try:
|
||||
for req in pip.req.parse_requirements(filename, options=options):
|
||||
req = install_requirement_ensure_req_field(req)
|
||||
all_requirements.setdefault(req.req.key, []).append(req)
|
||||
all_requirements[req.req.key].append(req)
|
||||
except Exception as ex:
|
||||
logger.error("Cannot parse `%s': %s" % (filename, ex))
|
||||
sys.exit(BAD_REQUIREMENTS)
|
||||
# CLI provided files after that.
|
||||
for req_spec in options.requirement_specs:
|
||||
try:
|
||||
req = install_requirement_parse(req_spec, "command line")
|
||||
all_requirements[req.req.key].append(req)
|
||||
except Exception as ex:
|
||||
logger.error("Cannot parse `%s': %s" % (req_spec, ex))
|
||||
sys.exit(BAD_REQUIREMENTS)
|
||||
ignored_requirements = []
|
||||
for req_spec in options.ignore_packages:
|
||||
try:
|
||||
@ -249,6 +249,7 @@ def join_one_requirement(req_list):
|
||||
|
||||
def join_requirements(options):
|
||||
global joined_requirements
|
||||
|
||||
all_requirements, ignored_requirements = parse_requirements(options)
|
||||
skip_keys = set(pkg.req.key for pkg in ignored_requirements)
|
||||
installed_by_key = {}
|
||||
@ -275,7 +276,7 @@ def join_requirements(options):
|
||||
"%s>=%s" % (installed_req.project_name,
|
||||
installed_req.specs[0][1]),
|
||||
"pip freeze")
|
||||
incompatible_requirement(frozen_req, joined_req)
|
||||
add_incompatible_requirement(frozen_req, joined_req)
|
||||
joined_req = frozen_req
|
||||
joined_requirements.append(joined_req.req)
|
||||
|
||||
@ -297,26 +298,36 @@ def join_requirements(options):
|
||||
if exact_version:
|
||||
for req in req_list:
|
||||
if not exact_version in req.req:
|
||||
incompatible_requirement(joined_req, req)
|
||||
add_incompatible_requirement(joined_req, req)
|
||||
else:
|
||||
for req in req_list:
|
||||
for parsed, trans, op, ver in req.req.index:
|
||||
if op[0] == "=":
|
||||
if parsed in conflicts:
|
||||
incompatible_requirement(joined_req, req)
|
||||
add_incompatible_requirement(joined_req, req)
|
||||
break
|
||||
elif not segment_ok and op[0] == "<":
|
||||
# analyse lower bound: x >= A or x > A
|
||||
if (lower_version > parsed or (
|
||||
lower_version == parsed and
|
||||
(lower_strict or len(op) != 2))):
|
||||
incompatible_requirement(joined_req, req)
|
||||
add_incompatible_requirement(joined_req, req)
|
||||
break
|
||||
|
||||
|
||||
def print_requirements():
|
||||
for req in sorted(joined_requirements, key=lambda x: x.key):
|
||||
print req
|
||||
global joined_requirements
|
||||
global incompatibles
|
||||
|
||||
requirements = {
|
||||
'compatibles': [],
|
||||
'incompatibles': {},
|
||||
}
|
||||
for r in joined_requirements:
|
||||
if r.key not in incompatibles:
|
||||
requirements['compatibles'].append(str(r))
|
||||
requirements['incompatibles'].update(incompatibles)
|
||||
print(json.dumps(requirements))
|
||||
|
||||
|
||||
def main():
|
||||
@ -325,8 +336,6 @@ def main():
|
||||
setup_logging(options)
|
||||
join_requirements(options)
|
||||
print_requirements()
|
||||
if incompatibles:
|
||||
sys.exit(INCOMPATIBLE_REQUIREMENTS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user