x/anvil
Code Issues Proposed changes

Merge "Unpin the pip version"

Browse Source
This commit is contained in:
Jenkins 2015-09-08 17:39:14 +00:00 committed by Gerrit Code Review
commit dda9b30745
6 changed files with 87 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
anvil/packaging/base.py
View File

@ -18,6 +18,8 @@
# R0921: Abstract class not referenced
#pylint: disable=R0902,R0921
import functools
from anvil import colorizer
from anvil import exceptions as exc
from anvil import log as logging
@ -180,36 +182,9 @@ class DependencyHandler(object):
sh.unlink(self.tracereader.filename())
def _scan_pip_requires(self, requires_files):
own_eggs = self._python_eggs(False)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs(False):
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in sorted(requires_files):
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
@ -237,12 +212,52 @@ class DependencyHandler(object):
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
return len(alterations)
def on_replace_done(fn, time_taken):
LOG.debug("Replacing potential forced requirements in %s"
" took %s seconds", colorizer.quote(fn), time_taken)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in own_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
requires_files = sorted(requires_files)
utils.log_iterable(requires_files,
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in requires_files:
LOG.debug("Replacing any potential forced requirements in %s",
colorizer.quote(fn))
mutations += utils.time_it(functools.partial(on_replace_done, fn),
replace_forced_requirements,
fn, forced_by_key)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:

29
anvil/packaging/helpers/pip_helper.py
View File

@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from distutils import version as dist_version
import glob
import pkg_resources
import re
@ -22,16 +21,13 @@ import sys
import tempfile
import threading
from pip import download as pip_download
from pip import req as pip_req
from pip import utils as pip_util
import pkginfo
import six
try:
from pip import util as pip_util
except ImportError:
# pip >=6 changed this location for some reason...
from pip import utils as pip_util
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@ -45,7 +41,6 @@ EGGS_DETAILED = {}
EGGS_DETAILED_LOCK = threading.RLock()
PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
PIP_VERSION = pkg_resources.get_distribution('pip').version
PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python'])
@ -208,6 +203,7 @@ def parse_requirements(contents):
def read_requirement_files(files):
pip_requirements = []
session = pip_download.PipSession()
for filename in files:
if sh.isfile(filename):
cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
@ -215,7 +211,8 @@ def read_requirement_files(files):
try:
reqs = REQUIREMENT_FILE_CACHE[cache_key]
except KeyError:
reqs = tuple(pip_req.parse_requirements(filename))
reqs = tuple(pip_req.parse_requirements(filename,
session=session))
REQUIREMENT_FILE_CACHE[cache_key] = reqs
pip_requirements.extend(reqs)
return (pip_requirements,
@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename):
if sh.isdir(build_path):
sh.deldir(build_path)
sh.mkdir(build_path)
# Ensure certain directories exist that we want to exist (but we don't
# want to delete them run after run).
cache_path = sh.joinpths(download_dir, ".cache")
if not sh.isdir(cache_path):
sh.mkdir(cache_path)
cmdline = [
PIP_EXECUTABLE, '-v',
'install', '-I', '-U',
'--download', download_dir,
'--build', build_path,
'--download-cache', cache_path,
# Don't download wheels since we lack the ability to create
# rpms from them (until future when we will have it, if ever)...
"--no-use-wheel",
]
# Don't download wheels...
#
# See: https://github.com/pypa/pip/issues/1439
if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
cmdline.append("--no-use-wheel")
for p in pips_to_download:
for p_seg in _split(p):
if p_seg:

6
anvil/packaging/venv.py
View File

@ -58,7 +58,6 @@ class VenvDependencyHandler(base.DependencyHandler):
super(VenvDependencyHandler, self).__init__(distro, root_dir,
instances, opts, group,
prior_groups)
self.cache_dir = sh.joinpths(self.root_dir, "pip-cache")
self.jobs = max(0, int(opts.get('jobs', 0)))
self.install_counters = {}
@ -76,14 +75,9 @@ class VenvDependencyHandler(base.DependencyHandler):
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
cmd.extend([
'--download-cache',
self.cache_dir,
])
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',

2
requirements.txt
View File

@ -2,7 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration
# process (and later running processes...).
pip<6
pip>=6
pkginfo
cheetah>=2.4.4
iniparse

56
tools/multipip
View File

@ -5,16 +5,15 @@ from __future__ import print_function
import collections
import itertools
import logging
import pkg_resources
import re
import sys
import argparse
import six
import pip.index
import pip.req
import pkg_resources
from pip import download as pip_download
from pip import req as pip_req
# Use this for the sorting order of operators
OP_ORDER = ('!=', '==', '<', '<=', '>', '>=')
@ -84,21 +83,6 @@ def setup_logging(options):
LOGGER.setLevel(level)
def install_requirement_ensure_req_field(req):
if not hasattr(req, 'req') or not req.req:
# pip 0.8 or so
link = pip.index.Link(req.url)
name = link.egg_fragment
if not name:
raise Exception("Cannot find package name from `%s'" % req.url)
req.req = pkg_resources.Requirement.parse(name)
return req
def install_requirement_str(req):
return req.url or str(req.req)
def install_requirement_parse(line, comes_from):
line = line.strip()
if line.startswith('-e') or line.startswith('--editable'):
@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
req = pip.req.InstallRequirement.from_editable(
req = pip_req.InstallRequirement.from_editable(
line, comes_from=comes_from)
else:
req = pip.req.InstallRequirement.from_line(line, comes_from)
return install_requirement_ensure_req_field(req)
req = pip_req.InstallRequirement.from_line(line, comes_from)
return req
def iter_combinations(elements, include_empty=False):
@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None):
for (op, version) in specs:
spec_pieces.append("%s%s" % (op, version))
spec = "%s%s" % (req_key, ",".join(spec_pieces))
sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled'))
sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled'))
return sources
def reform_incompatibles(incompatible_specs, versions):
@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None):
if not matches:
spec_pieces = "%s%s" % (op, version)
spec = "%s%s" % (req_key, spec_pieces)
causes.append(pip.req.InstallRequirement.from_line(spec,
causes.append(pip_req.InstallRequirement.from_line(spec,
"compiled conflict"))
return causes
@ -370,10 +354,10 @@ def parse_requirements(options):
all_requirements.setdefault(req_key(req), []).append(req)
except Exception as ex:
raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex))
session = pip_download.PipSession()
for filename in options.requirements:
try:
for req in pip.req.parse_requirements(filename):
req = install_requirement_ensure_req_field(req)
for req in pip_req.parse_requirements(filename, session=session):
if skip_match and skip_match.search(req_key(req)):
continue
all_requirements.setdefault(req_key(req), []).append(req)
@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements):
return (joined_requirements, incompatibles)
def reform_req(req):
if req.editable:
return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req)
else:
return str(req.req)
def print_requirements(joined_requirements):
formatted_requirements = []
for req_key in sorted(six.iterkeys(joined_requirements)):
req = joined_requirements[req_key][0]
req_prefix = ""
if req.editable:
req_prefix = "-e "
if req.url:
req = "%s#egg=%s" % (req.url, req.req)
else:
req = str(req.req)
formatted_requirements.append("%s%s" % (req_prefix, req))
formatted_requirements.append(reform_req(req))
for req in formatted_requirements:
print(req)
@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements):
print("Choosing:", file=sys.stderr)
for chosen in chosen_reqs:
print("\t%s: %s" % (chosen.comes_from,
install_requirement_str(chosen)),
reform_req(chosen)),
file=sys.stderr)
print("Conflicting:", file=sys.stderr)
for conflicting in req_incompatibles:
print("\t%s: %s" % (conflicting.comes_from,
install_requirement_str(conflicting)),
reform_req(conflicting)),
file=sys.stderr)

15
tools/py2rpm
View File

@ -15,9 +15,9 @@ import sys
import tempfile
import argparse
from pip import utils as pip_util
import six
import pip.util
import pkg_resources
@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs):
def build_rpm(options, filename, build_options):
if os.path.isfile(filename):
temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
pip.util.unpack_file(filename, temp_dir, None, None)
pip_util.unpack_file(filename, temp_dir, None, None)
source_dir = temp_dir
archive_name = filename
elif os.path.isdir(filename):
@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options):
pkg_key = python_name_to_key(pkg_name)
build_dir = options.rpm_base
rpm_name = python_key_to_rpm(pkg_key)
version = one_line(info["version"])
# NOTE(harlowja): try not to use info["version"] to get the version, since
# currently that is getting normalized by setuptools to be a normalized
# version which can be different from the actual version...
try:
version = setup_py_one_line(source_dir, "--version")
except Exception:
version = info['version']
logger.warning("Failed extracting version, falling back to '%s'",
version, exc_info=True)
cleaned_version = version.replace('-', '_')
spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)