x/anvil
Code Issues Proposed changes

Unpin the pip version

Browse Source 
We were restricted to older versions of pip due to some
import changes, and code changes the pip folks did, but this
changes that so that we can now use the newer versions of
pip and all the features/additions they have performed there.

Change-Id: Ia4bff0229acf7823c8fe9e0e95bf3ac02f9a6ce1
This commit is contained in:
Joshua Harlow 2015-07-09 17:06:41 -07:00
parent 062ecc164c
commit 82671430e6
6 changed files with 87 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
anvil/packaging/base.py
View File

@ -18,6 +18,8 @@
# R0921: Abstract class not referenced
#pylint: disable=R0902,R0921
import functools
from anvil import colorizer
from anvil import exceptions as exc
from anvil import log as logging
@ -175,36 +177,9 @@ class DependencyHandler(object):
sh.unlink(self.tracereader.filename())
def _scan_pip_requires(self, requires_files):
own_eggs = self._python_eggs(False)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in self._python_eggs(False):
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
utils.log_iterable(sorted(requires_files),
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in sorted(requires_files):
def replace_forced_requirements(fn, forced_by_key):
old_lines = sh.load_file(fn).splitlines()
new_lines = []
alterations = []
@ -232,12 +207,52 @@ class DependencyHandler(object):
if alterations:
contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
sh.write_file_and_backup(fn, contents)
mutations += len(alterations)
utils.log_iterable(alterations,
logger=LOG,
header="Replaced %s requirements in %s"
% (len(alterations), fn),
color=None)
return len(alterations)
def on_replace_done(fn, time_taken):
LOG.debug("Replacing potential forced requirements in %s"
" took %s seconds", colorizer.quote(fn), time_taken)
def validate_requirement(filename, source_req):
install_egg = None
for egg_info in own_eggs:
if egg_info['name'] == source_req.key:
install_egg = egg_info
break
if not install_egg:
return
# Ensure what we are about to install/create will actually work
# with the desired version. If it is not compatible then we should
# abort and someone should update the tag/branch in the origin
# file (or fix it via some other mechanism).
if install_egg['version'] not in source_req:
msg = ("Can not satisfy '%s' with '%s', version"
" conflict found in %s")
raise exc.DependencyException(msg % (source_req,
install_egg['req'],
filename))
if not requires_files:
return
requires_files = sorted(requires_files)
utils.log_iterable(requires_files,
logger=LOG,
header="Scanning %s pip 'requires' files" % (len(requires_files)))
forced_by_key = {}
for pkg in self.forced_pips:
forced_by_key[pkg.key] = pkg
mutations = 0
for fn in requires_files:
LOG.debug("Replacing any potential forced requirements in %s",
colorizer.quote(fn))
mutations += utils.time_it(functools.partial(on_replace_done, fn),
replace_forced_requirements,
fn, forced_by_key)
# NOTE(imelnikov): after updating requirement lists we should re-fetch
# data from them again, so we drop pip helper caches here.
if mutations > 0:

29
anvil/packaging/helpers/pip_helper.py
View File

@ -14,7 +14,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from distutils import version as dist_version
import glob
import pkg_resources
import re
@ -22,16 +21,13 @@ import sys
import tempfile
import threading
from pip import download as pip_download
from pip import req as pip_req
from pip import utils as pip_util
import pkginfo
import six
try:
from pip import util as pip_util
except ImportError:
# pip >=6 changed this location for some reason...
from pip import utils as pip_util
from anvil import log as logging
from anvil import shell as sh
from anvil import utils
@ -45,7 +41,6 @@ EGGS_DETAILED = {}
EGGS_DETAILED_LOCK = threading.RLock()
PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
PIP_VERSION = pkg_resources.get_distribution('pip').version
PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python'])
@ -208,6 +203,7 @@ def parse_requirements(contents):
def read_requirement_files(files):
pip_requirements = []
session = pip_download.PipSession()
for filename in files:
if sh.isfile(filename):
cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
@ -215,7 +211,8 @@ def read_requirement_files(files):
try:
reqs = REQUIREMENT_FILE_CACHE[cache_key]
except KeyError:
reqs = tuple(pip_req.parse_requirements(filename))
reqs = tuple(pip_req.parse_requirements(filename,
session=session))
REQUIREMENT_FILE_CACHE[cache_key] = reqs
pip_requirements.extend(reqs)
return (pip_requirements,
@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename):
if sh.isdir(build_path):
sh.deldir(build_path)
sh.mkdir(build_path)
# Ensure certain directories exist that we want to exist (but we don't
# want to delete them run after run).
cache_path = sh.joinpths(download_dir, ".cache")
if not sh.isdir(cache_path):
sh.mkdir(cache_path)
cmdline = [
PIP_EXECUTABLE, '-v',
'install', '-I', '-U',
'--download', download_dir,
'--build', build_path,
'--download-cache', cache_path,
# Don't download wheels since we lack the ability to create
# rpms from them (until future when we will have it, if ever)...
"--no-use-wheel",
]
# Don't download wheels...
#
# See: https://github.com/pypa/pip/issues/1439
if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
cmdline.append("--no-use-wheel")
for p in pips_to_download:
for p_seg in _split(p):
if p_seg:

6
anvil/packaging/venv.py
View File

@ -64,7 +64,6 @@ class VenvDependencyHandler(base.DependencyHandler):
super(VenvDependencyHandler, self).__init__(distro, root_dir,
instances, opts, group,
prior_groups)
self.cache_dir = sh.joinpths(self.root_dir, "pip-cache")
self.jobs = max(0, int(opts.get('jobs', 0)))
def _venv_directory_for(self, instance):
@ -81,14 +80,9 @@ class VenvDependencyHandler(base.DependencyHandler):
}
if extra_env_overrides:
env_overrides.update(extra_env_overrides)
sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
cmd = list(base_pip) + ['install']
if upgrade:
cmd.append("--upgrade")
cmd.extend([
'--download-cache',
self.cache_dir,
])
if isinstance(requirements, six.string_types):
cmd.extend([
'--requirement',

2
requirements.txt
View File

@ -2,7 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration
# process (and later running processes...).
pip<6
pip>=6
pkginfo
cheetah>=2.4.4
iniparse

56
tools/multipip
View File

@ -5,16 +5,15 @@ from __future__ import print_function
import collections
import itertools
import logging
import pkg_resources
import re
import sys
import argparse
import six
import pip.index
import pip.req
import pkg_resources
from pip import download as pip_download
from pip import req as pip_req
# Use this for the sorting order of operators
OP_ORDER = ('!=', '==', '<', '<=', '>', '>=')
@ -84,21 +83,6 @@ def setup_logging(options):
LOGGER.setLevel(level)
def install_requirement_ensure_req_field(req):
if not hasattr(req, 'req') or not req.req:
# pip 0.8 or so
link = pip.index.Link(req.url)
name = link.egg_fragment
if not name:
raise Exception("Cannot find package name from `%s'" % req.url)
req.req = pkg_resources.Requirement.parse(name)
return req
def install_requirement_str(req):
return req.url or str(req.req)
def install_requirement_parse(line, comes_from):
line = line.strip()
if line.startswith('-e') or line.startswith('--editable'):
@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
req = pip.req.InstallRequirement.from_editable(
req = pip_req.InstallRequirement.from_editable(
line, comes_from=comes_from)
else:
req = pip.req.InstallRequirement.from_line(line, comes_from)
return install_requirement_ensure_req_field(req)
req = pip_req.InstallRequirement.from_line(line, comes_from)
return req
def iter_combinations(elements, include_empty=False):
@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None):
for (op, version) in specs:
spec_pieces.append("%s%s" % (op, version))
spec = "%s%s" % (req_key, ",".join(spec_pieces))
sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled'))
sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled'))
return sources
def reform_incompatibles(incompatible_specs, versions):
@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None):
if not matches:
spec_pieces = "%s%s" % (op, version)
spec = "%s%s" % (req_key, spec_pieces)
causes.append(pip.req.InstallRequirement.from_line(spec,
causes.append(pip_req.InstallRequirement.from_line(spec,
"compiled conflict"))
return causes
@ -370,10 +354,10 @@ def parse_requirements(options):
all_requirements.setdefault(req_key(req), []).append(req)
except Exception as ex:
raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex))
session = pip_download.PipSession()
for filename in options.requirements:
try:
for req in pip.req.parse_requirements(filename):
req = install_requirement_ensure_req_field(req)
for req in pip_req.parse_requirements(filename, session=session):
if skip_match and skip_match.search(req_key(req)):
continue
all_requirements.setdefault(req_key(req), []).append(req)
@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements):
return (joined_requirements, incompatibles)
def reform_req(req):
if req.editable:
return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req)
else:
return str(req.req)
def print_requirements(joined_requirements):
formatted_requirements = []
for req_key in sorted(six.iterkeys(joined_requirements)):
req = joined_requirements[req_key][0]
req_prefix = ""
if req.editable:
req_prefix = "-e "
if req.url:
req = "%s#egg=%s" % (req.url, req.req)
else:
req = str(req.req)
formatted_requirements.append("%s%s" % (req_prefix, req))
formatted_requirements.append(reform_req(req))
for req in formatted_requirements:
print(req)
@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements):
print("Choosing:", file=sys.stderr)
for chosen in chosen_reqs:
print("\t%s: %s" % (chosen.comes_from,
install_requirement_str(chosen)),
reform_req(chosen)),
file=sys.stderr)
print("Conflicting:", file=sys.stderr)
for conflicting in req_incompatibles:
print("\t%s: %s" % (conflicting.comes_from,
install_requirement_str(conflicting)),
reform_req(conflicting)),
file=sys.stderr)

15
tools/py2rpm
View File

@ -15,9 +15,9 @@ import sys
import tempfile
import argparse
from pip import utils as pip_util
import six
import pip.util
import pkg_resources
@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs):
def build_rpm(options, filename, build_options):
if os.path.isfile(filename):
temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
pip.util.unpack_file(filename, temp_dir, None, None)
pip_util.unpack_file(filename, temp_dir, None, None)
source_dir = temp_dir
archive_name = filename
elif os.path.isdir(filename):
@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options):
pkg_key = python_name_to_key(pkg_name)
build_dir = options.rpm_base
rpm_name = python_key_to_rpm(pkg_key)
version = one_line(info["version"])
# NOTE(harlowja): try not to use info["version"] to get the version, since
# currently that is getting normalized by setuptools to be a normalized
# version which can be different from the actual version...
try:
version = setup_py_one_line(source_dir, "--version")
except Exception:
version = info['version']
logger.warning("Failed extracting version, falling back to '%s'",
version, exc_info=True)
cleaned_version = version.replace('-', '_')
spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)