Unpin the pip version

We were restricted to older versions of pip due to some
import changes, and code changes the pip folks did, but this
changes that so that we can now use the newer versions of
pip and all the features/additions they have performed there.

Change-Id: Ia4bff0229acf7823c8fe9e0e95bf3ac02f9a6ce1

anvil/packaging/base.py

@@ -18,6 +18,8 @@
 # R0921: Abstract class not referenced
 #pylint: disable=R0902,R0921
 
+import functools
+
 from anvil import colorizer
 from anvil import exceptions as exc
 from anvil import log as logging
@@ -175,36 +177,9 @@ class DependencyHandler(object):
             sh.unlink(self.tracereader.filename())
 
     def _scan_pip_requires(self, requires_files):
+        own_eggs = self._python_eggs(False)
 
-        def validate_requirement(filename, source_req):
+        def replace_forced_requirements(fn, forced_by_key):
-            install_egg = None
-            for egg_info in self._python_eggs(False):
-                if egg_info['name'] == source_req.key:
-                    install_egg = egg_info
-                    break
-            if not install_egg:
-                return
-            # Ensure what we are about to install/create will actually work
-            # with the desired version. If it is not compatible then we should
-            # abort and someone should update the tag/branch in the origin
-            # file (or fix it via some other mechanism).
-            if install_egg['version'] not in source_req:
-                msg = ("Can not satisfy '%s' with '%s', version"
-                       " conflict found in %s")
-                raise exc.DependencyException(msg % (source_req,
-                                                     install_egg['req'],
-                                                     filename))
-
-        if not requires_files:
-            return
-        utils.log_iterable(sorted(requires_files),
-                           logger=LOG,
-                           header="Scanning %s pip 'requires' files" % (len(requires_files)))
-        forced_by_key = {}
-        for pkg in self.forced_pips:
-            forced_by_key[pkg.key] = pkg
-        mutations = 0
-        for fn in sorted(requires_files):
             old_lines = sh.load_file(fn).splitlines()
             new_lines = []
             alterations = []
@@ -232,12 +207,52 @@ class DependencyHandler(object):
             if alterations:
                 contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
                 sh.write_file_and_backup(fn, contents)
-                mutations += len(alterations)
                 utils.log_iterable(alterations,
                                    logger=LOG,
                                    header="Replaced %s requirements in %s"
                                           % (len(alterations), fn),
                                    color=None)
+            return len(alterations)
+
+        def on_replace_done(fn, time_taken):
+            LOG.debug("Replacing potential forced requirements in %s"
+                      " took %s seconds", colorizer.quote(fn), time_taken)
+
+        def validate_requirement(filename, source_req):
+            install_egg = None
+            for egg_info in own_eggs:
+                if egg_info['name'] == source_req.key:
+                    install_egg = egg_info
+                    break
+            if not install_egg:
+                return
+            # Ensure what we are about to install/create will actually work
+            # with the desired version. If it is not compatible then we should
+            # abort and someone should update the tag/branch in the origin
+            # file (or fix it via some other mechanism).
+            if install_egg['version'] not in source_req:
+                msg = ("Can not satisfy '%s' with '%s', version"
+                       " conflict found in %s")
+                raise exc.DependencyException(msg % (source_req,
+                                                     install_egg['req'],
+                                                     filename))
+
+        if not requires_files:
+            return
+        requires_files = sorted(requires_files)
+        utils.log_iterable(requires_files,
+                           logger=LOG,
+                           header="Scanning %s pip 'requires' files" % (len(requires_files)))
+        forced_by_key = {}
+        for pkg in self.forced_pips:
+            forced_by_key[pkg.key] = pkg
+        mutations = 0
+        for fn in requires_files:
+            LOG.debug("Replacing any potential forced requirements in %s",
+                      colorizer.quote(fn))
+            mutations += utils.time_it(functools.partial(on_replace_done, fn),
+                                       replace_forced_requirements,
+                                       fn, forced_by_key)
         # NOTE(imelnikov): after updating requirement lists we should re-fetch
         # data from them again, so we drop pip helper caches here.
         if mutations > 0:

anvil/packaging/helpers/pip_helper.py

@@ -14,7 +14,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from distutils import version as dist_version
 import glob
 import pkg_resources
 import re
@@ -22,16 +21,13 @@ import sys
 import tempfile
 import threading
 
+from pip import download as pip_download
 from pip import req as pip_req
+from pip import utils as pip_util
+
 import pkginfo
 import six
 
-try:
-    from pip import util as pip_util
-except ImportError:
-    # pip >=6 changed this location for some reason...
-    from pip import utils as pip_util
-
 from anvil import log as logging
 from anvil import shell as sh
 from anvil import utils
@@ -45,7 +41,6 @@ EGGS_DETAILED = {}
 EGGS_DETAILED_LOCK = threading.RLock()
 
 PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
-PIP_VERSION = pkg_resources.get_distribution('pip').version
 PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python'])
 
 
@@ -208,6 +203,7 @@ def parse_requirements(contents):
 
 def read_requirement_files(files):
     pip_requirements = []
+    session = pip_download.PipSession()
     for filename in files:
         if sh.isfile(filename):
             cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
@@ -215,7 +211,8 @@ def read_requirement_files(files):
                 try:
                     reqs = REQUIREMENT_FILE_CACHE[cache_key]
                 except KeyError:
-                    reqs = tuple(pip_req.parse_requirements(filename))
+                    reqs = tuple(pip_req.parse_requirements(filename,
+                                                            session=session))
                     REQUIREMENT_FILE_CACHE[cache_key] = reqs
                 pip_requirements.extend(reqs)
     return (pip_requirements,
@@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename):
     if sh.isdir(build_path):
         sh.deldir(build_path)
     sh.mkdir(build_path)
-    # Ensure certain directories exist that we want to exist (but we don't
-    # want to delete them run after run).
-    cache_path = sh.joinpths(download_dir, ".cache")
-    if not sh.isdir(cache_path):
-        sh.mkdir(cache_path)
     cmdline = [
         PIP_EXECUTABLE, '-v',
         'install', '-I', '-U',
         '--download', download_dir,
         '--build', build_path,
-        '--download-cache', cache_path,
+        # Don't download wheels since we lack the ability to create
+        # rpms from them (until future when we will have it, if ever)...
+        "--no-use-wheel",
     ]
-    # Don't download wheels...
-    #
-    # See: https://github.com/pypa/pip/issues/1439
-    if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
-        cmdline.append("--no-use-wheel")
     for p in pips_to_download:
         for p_seg in _split(p):
             if p_seg:

anvil/packaging/venv.py

@@ -64,7 +64,6 @@ class VenvDependencyHandler(base.DependencyHandler):
         super(VenvDependencyHandler, self).__init__(distro, root_dir,
                                                     instances, opts, group,
                                                     prior_groups)
-        self.cache_dir = sh.joinpths(self.root_dir, "pip-cache")
         self.jobs = max(0, int(opts.get('jobs', 0)))
 
     def _venv_directory_for(self, instance):
@@ -81,14 +80,9 @@ class VenvDependencyHandler(base.DependencyHandler):
         }
         if extra_env_overrides:
             env_overrides.update(extra_env_overrides)
-        sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
         cmd = list(base_pip) + ['install']
         if upgrade:
             cmd.append("--upgrade")
-        cmd.extend([
-            '--download-cache',
-            self.cache_dir,
-        ])
         if isinstance(requirements, six.string_types):
             cmd.extend([
                 '--requirement',

requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process (and later running processes...).
 
-pip<6
+pip>=6
 pkginfo
 cheetah>=2.4.4
 iniparse

tools/multipip

@@ -5,16 +5,15 @@ from __future__ import print_function
 import collections
 import itertools
 import logging
+import pkg_resources
 import re
 import sys
 
 import argparse
 import six
 
-import pip.index
+from pip import download as pip_download
-import pip.req
+from pip import req as pip_req
-import pkg_resources
-
 
 # Use this for the sorting order of operators
 OP_ORDER = ('!=', '==', '<', '<=', '>', '>=')
@@ -84,21 +83,6 @@ def setup_logging(options):
     LOGGER.setLevel(level)
 
 
-def install_requirement_ensure_req_field(req):
-    if not hasattr(req, 'req') or not req.req:
-        # pip 0.8 or so
-        link = pip.index.Link(req.url)
-        name = link.egg_fragment
-        if not name:
-            raise Exception("Cannot find package name from `%s'" % req.url)
-        req.req = pkg_resources.Requirement.parse(name)
-    return req
-
-
-def install_requirement_str(req):
-    return req.url or str(req.req)
-
-
 def install_requirement_parse(line, comes_from):
     line = line.strip()
     if line.startswith('-e') or line.startswith('--editable'):
@@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from):
             line = line[2:].strip()
         else:
             line = line[len('--editable'):].strip().lstrip('=')
-        req = pip.req.InstallRequirement.from_editable(
+        req = pip_req.InstallRequirement.from_editable(
             line, comes_from=comes_from)
     else:
-        req = pip.req.InstallRequirement.from_line(line, comes_from)
+        req = pip_req.InstallRequirement.from_line(line, comes_from)
-    return install_requirement_ensure_req_field(req)
+    return req
 
 
 def iter_combinations(elements, include_empty=False):
@@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None):
             for (op, version) in specs:
                 spec_pieces.append("%s%s" % (op, version))
             spec = "%s%s" % (req_key, ",".join(spec_pieces))
-            sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled'))
+            sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled'))
         return sources
 
     def reform_incompatibles(incompatible_specs, versions):
@@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None):
             if not matches:
                 spec_pieces = "%s%s" % (op, version)
                 spec = "%s%s" % (req_key, spec_pieces)
-                causes.append(pip.req.InstallRequirement.from_line(spec,
+                causes.append(pip_req.InstallRequirement.from_line(spec,
                                                                    "compiled conflict"))
         return causes
 
@@ -370,10 +354,10 @@ def parse_requirements(options):
             all_requirements.setdefault(req_key(req), []).append(req)
         except Exception as ex:
             raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex))
+    session = pip_download.PipSession()
     for filename in options.requirements:
         try:
-            for req in pip.req.parse_requirements(filename):
+            for req in pip_req.parse_requirements(filename, session=session):
-                req = install_requirement_ensure_req_field(req)
                 if skip_match and skip_match.search(req_key(req)):
                     continue
                 all_requirements.setdefault(req_key(req), []).append(req)
@@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements):
     return (joined_requirements, incompatibles)
 
 
+def reform_req(req):
+    if req.editable:
+        return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req)
+    else:
+        return str(req.req)
+
+
 def print_requirements(joined_requirements):
     formatted_requirements = []
     for req_key in sorted(six.iterkeys(joined_requirements)):
         req = joined_requirements[req_key][0]
-        req_prefix = ""
+        formatted_requirements.append(reform_req(req))
-        if req.editable:
-            req_prefix = "-e "
-        if req.url:
-            req = "%s#egg=%s" % (req.url, req.req)
-        else:
-            req = str(req.req)
-        formatted_requirements.append("%s%s" % (req_prefix, req))
     for req in formatted_requirements:
         print(req)
 
@@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements):
             print("Choosing:", file=sys.stderr)
             for chosen in chosen_reqs:
                 print("\t%s: %s" % (chosen.comes_from,
-                                    install_requirement_str(chosen)),
+                                    reform_req(chosen)),
                       file=sys.stderr)
         print("Conflicting:", file=sys.stderr)
         for conflicting in req_incompatibles:
             print("\t%s: %s" % (conflicting.comes_from,
-                                install_requirement_str(conflicting)),
+                                reform_req(conflicting)),
                   file=sys.stderr)
 
 

tools/py2rpm

@@ -15,9 +15,9 @@ import sys
 import tempfile
 
 import argparse
+from pip import utils as pip_util
 import six
 
-import pip.util
 import pkg_resources
 
 
@@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs):
 def build_rpm(options, filename, build_options):
     if os.path.isfile(filename):
         temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
-        pip.util.unpack_file(filename, temp_dir, None, None)
+        pip_util.unpack_file(filename, temp_dir, None, None)
         source_dir = temp_dir
         archive_name = filename
     elif os.path.isdir(filename):
@@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options):
     pkg_key = python_name_to_key(pkg_name)
     build_dir = options.rpm_base
     rpm_name = python_key_to_rpm(pkg_key)
-    version = one_line(info["version"])
+
+    # NOTE(harlowja): try not to use info["version"] to get the version, since
+    # currently that is getting normalized by setuptools to be a normalized
+    # version which can be different from the actual version...
+    try:
+        version = setup_py_one_line(source_dir, "--version")
+    except Exception:
+        version = info['version']
+        logger.warning("Failed extracting version, falling back to '%s'",
+                       version, exc_info=True)
     cleaned_version = version.replace('-', '_')
     spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)