Merge "Unpin the pip version"

anvil/packaging/base.py

@@ -18,6 +18,8 @@
 # R0921: Abstract class not referenced
 #pylint: disable=R0902,R0921
 
+import functools
+
 from anvil import colorizer
 from anvil import exceptions as exc
 from anvil import log as logging
@@ -180,36 +182,9 @@ class DependencyHandler(object):
             sh.unlink(self.tracereader.filename())
 
     def _scan_pip_requires(self, requires_files):
+        own_eggs = self._python_eggs(False)
 
-        def validate_requirement(filename, source_req):
+        def replace_forced_requirements(fn, forced_by_key):
-            install_egg = None
-            for egg_info in self._python_eggs(False):
-                if egg_info['name'] == source_req.key:
-                    install_egg = egg_info
-                    break
-            if not install_egg:
-                return
-            # Ensure what we are about to install/create will actually work
-            # with the desired version. If it is not compatible then we should
-            # abort and someone should update the tag/branch in the origin
-            # file (or fix it via some other mechanism).
-            if install_egg['version'] not in source_req:
-                msg = ("Can not satisfy '%s' with '%s', version"
-                       " conflict found in %s")
-                raise exc.DependencyException(msg % (source_req,
-                                                     install_egg['req'],
-                                                     filename))
-
-        if not requires_files:
-            return
-        utils.log_iterable(sorted(requires_files),
-                           logger=LOG,
-                           header="Scanning %s pip 'requires' files" % (len(requires_files)))
-        forced_by_key = {}
-        for pkg in self.forced_pips:
-            forced_by_key[pkg.key] = pkg
-        mutations = 0
-        for fn in sorted(requires_files):
             old_lines = sh.load_file(fn).splitlines()
             new_lines = []
             alterations = []
@@ -237,12 +212,52 @@ class DependencyHandler(object):
             if alterations:
                 contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines))
                 sh.write_file_and_backup(fn, contents)
-                mutations += len(alterations)
                 utils.log_iterable(alterations,
                                    logger=LOG,
                                    header="Replaced %s requirements in %s"
                                           % (len(alterations), fn),
                                    color=None)
+            return len(alterations)
+
+        def on_replace_done(fn, time_taken):
+            LOG.debug("Replacing potential forced requirements in %s"
+                      " took %s seconds", colorizer.quote(fn), time_taken)
+
+        def validate_requirement(filename, source_req):
+            install_egg = None
+            for egg_info in own_eggs:
+                if egg_info['name'] == source_req.key:
+                    install_egg = egg_info
+                    break
+            if not install_egg:
+                return
+            # Ensure what we are about to install/create will actually work
+            # with the desired version. If it is not compatible then we should
+            # abort and someone should update the tag/branch in the origin
+            # file (or fix it via some other mechanism).
+            if install_egg['version'] not in source_req:
+                msg = ("Can not satisfy '%s' with '%s', version"
+                       " conflict found in %s")
+                raise exc.DependencyException(msg % (source_req,
+                                                     install_egg['req'],
+                                                     filename))
+
+        if not requires_files:
+            return
+        requires_files = sorted(requires_files)
+        utils.log_iterable(requires_files,
+                           logger=LOG,
+                           header="Scanning %s pip 'requires' files" % (len(requires_files)))
+        forced_by_key = {}
+        for pkg in self.forced_pips:
+            forced_by_key[pkg.key] = pkg
+        mutations = 0
+        for fn in requires_files:
+            LOG.debug("Replacing any potential forced requirements in %s",
+                      colorizer.quote(fn))
+            mutations += utils.time_it(functools.partial(on_replace_done, fn),
+                                       replace_forced_requirements,
+                                       fn, forced_by_key)
         # NOTE(imelnikov): after updating requirement lists we should re-fetch
         # data from them again, so we drop pip helper caches here.
         if mutations > 0:

anvil/packaging/helpers/pip_helper.py

@@ -14,7 +14,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from distutils import version as dist_version
 import glob
 import pkg_resources
 import re
@@ -22,16 +21,13 @@ import sys
 import tempfile
 import threading
 
+from pip import download as pip_download
 from pip import req as pip_req
+from pip import utils as pip_util
+
 import pkginfo
 import six
 
-try:
-    from pip import util as pip_util
-except ImportError:
-    # pip >=6 changed this location for some reason...
-    from pip import utils as pip_util
-
 from anvil import log as logging
 from anvil import shell as sh
 from anvil import utils
@@ -45,7 +41,6 @@ EGGS_DETAILED = {}
 EGGS_DETAILED_LOCK = threading.RLock()
 
 PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$")
-PIP_VERSION = pkg_resources.get_distribution('pip').version
 PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python'])
 
 
@@ -208,6 +203,7 @@ def parse_requirements(contents):
 
 def read_requirement_files(files):
     pip_requirements = []
+    session = pip_download.PipSession()
     for filename in files:
         if sh.isfile(filename):
             cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename))
@@ -215,7 +211,8 @@ def read_requirement_files(files):
                 try:
                     reqs = REQUIREMENT_FILE_CACHE[cache_key]
                 except KeyError:
-                    reqs = tuple(pip_req.parse_requirements(filename))
+                    reqs = tuple(pip_req.parse_requirements(filename,
+                                                            session=session))
                     REQUIREMENT_FILE_CACHE[cache_key] = reqs
                 pip_requirements.extend(reqs)
     return (pip_requirements,
@@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename):
     if sh.isdir(build_path):
         sh.deldir(build_path)
     sh.mkdir(build_path)
-    # Ensure certain directories exist that we want to exist (but we don't
-    # want to delete them run after run).
-    cache_path = sh.joinpths(download_dir, ".cache")
-    if not sh.isdir(cache_path):
-        sh.mkdir(cache_path)
     cmdline = [
         PIP_EXECUTABLE, '-v',
         'install', '-I', '-U',
         '--download', download_dir,
         '--build', build_path,
-        '--download-cache', cache_path,
+        # Don't download wheels since we lack the ability to create
+        # rpms from them (until future when we will have it, if ever)...
+        "--no-use-wheel",
     ]
-    # Don't download wheels...
-    #
-    # See: https://github.com/pypa/pip/issues/1439
-    if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'):
-        cmdline.append("--no-use-wheel")
     for p in pips_to_download:
         for p_seg in _split(p):
             if p_seg:

anvil/packaging/venv.py

@@ -58,7 +58,6 @@ class VenvDependencyHandler(base.DependencyHandler):
         super(VenvDependencyHandler, self).__init__(distro, root_dir,
                                                     instances, opts, group,
                                                     prior_groups)
-        self.cache_dir = sh.joinpths(self.root_dir, "pip-cache")
         self.jobs = max(0, int(opts.get('jobs', 0)))
         self.install_counters = {}
 
@@ -76,14 +75,9 @@ class VenvDependencyHandler(base.DependencyHandler):
         }
         if extra_env_overrides:
             env_overrides.update(extra_env_overrides)
-        sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter)
         cmd = list(base_pip) + ['install']
         if upgrade:
             cmd.append("--upgrade")
-        cmd.extend([
-            '--download-cache',
-            self.cache_dir,
-        ])
         if isinstance(requirements, six.string_types):
             cmd.extend([
                 '--requirement',

requirements.txt

@@ -2,7 +2,7 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process (and later running processes...).
 
-pip<6
+pip>=6
 pkginfo
 cheetah>=2.4.4
 iniparse

tools/multipip

@@ -5,16 +5,15 @@ from __future__ import print_function
 import collections
 import itertools
 import logging
+import pkg_resources
 import re
 import sys
 
 import argparse
 import six
 
-import pip.index
+from pip import download as pip_download
-import pip.req
+from pip import req as pip_req
-import pkg_resources
-
 
 # Use this for the sorting order of operators
 OP_ORDER = ('!=', '==', '<', '<=', '>', '>=')
@@ -84,21 +83,6 @@ def setup_logging(options):
     LOGGER.setLevel(level)
 
 
-def install_requirement_ensure_req_field(req):
-    if not hasattr(req, 'req') or not req.req:
-        # pip 0.8 or so
-        link = pip.index.Link(req.url)
-        name = link.egg_fragment
-        if not name:
-            raise Exception("Cannot find package name from `%s'" % req.url)
-        req.req = pkg_resources.Requirement.parse(name)
-    return req
-
-
-def install_requirement_str(req):
-    return req.url or str(req.req)
-
-
 def install_requirement_parse(line, comes_from):
     line = line.strip()
     if line.startswith('-e') or line.startswith('--editable'):
@@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from):
             line = line[2:].strip()
         else:
             line = line[len('--editable'):].strip().lstrip('=')
-        req = pip.req.InstallRequirement.from_editable(
+        req = pip_req.InstallRequirement.from_editable(
             line, comes_from=comes_from)
     else:
-        req = pip.req.InstallRequirement.from_line(line, comes_from)
+        req = pip_req.InstallRequirement.from_line(line, comes_from)
-    return install_requirement_ensure_req_field(req)
+    return req
 
 
 def iter_combinations(elements, include_empty=False):
@@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None):
             for (op, version) in specs:
                 spec_pieces.append("%s%s" % (op, version))
             spec = "%s%s" % (req_key, ",".join(spec_pieces))
-            sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled'))
+            sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled'))
         return sources
 
     def reform_incompatibles(incompatible_specs, versions):
@@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None):
             if not matches:
                 spec_pieces = "%s%s" % (op, version)
                 spec = "%s%s" % (req_key, spec_pieces)
-                causes.append(pip.req.InstallRequirement.from_line(spec,
+                causes.append(pip_req.InstallRequirement.from_line(spec,
                                                                    "compiled conflict"))
         return causes
 
@@ -370,10 +354,10 @@ def parse_requirements(options):
             all_requirements.setdefault(req_key(req), []).append(req)
         except Exception as ex:
             raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex))
+    session = pip_download.PipSession()
     for filename in options.requirements:
         try:
-            for req in pip.req.parse_requirements(filename):
+            for req in pip_req.parse_requirements(filename, session=session):
-                req = install_requirement_ensure_req_field(req)
                 if skip_match and skip_match.search(req_key(req)):
                     continue
                 all_requirements.setdefault(req_key(req), []).append(req)
@@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements):
     return (joined_requirements, incompatibles)
 
 
+def reform_req(req):
+    if req.editable:
+        return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req)
+    else:
+        return str(req.req)
+
+
 def print_requirements(joined_requirements):
     formatted_requirements = []
     for req_key in sorted(six.iterkeys(joined_requirements)):
         req = joined_requirements[req_key][0]
-        req_prefix = ""
+        formatted_requirements.append(reform_req(req))
-        if req.editable:
-            req_prefix = "-e "
-        if req.url:
-            req = "%s#egg=%s" % (req.url, req.req)
-        else:
-            req = str(req.req)
-        formatted_requirements.append("%s%s" % (req_prefix, req))
     for req in formatted_requirements:
         print(req)
 
@@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements):
             print("Choosing:", file=sys.stderr)
             for chosen in chosen_reqs:
                 print("\t%s: %s" % (chosen.comes_from,
-                                    install_requirement_str(chosen)),
+                                    reform_req(chosen)),
                       file=sys.stderr)
         print("Conflicting:", file=sys.stderr)
         for conflicting in req_incompatibles:
             print("\t%s: %s" % (conflicting.comes_from,
-                                install_requirement_str(conflicting)),
+                                reform_req(conflicting)),
                   file=sys.stderr)
 
 

tools/py2rpm

@@ -15,9 +15,9 @@ import sys
 import tempfile
 
 import argparse
+from pip import utils as pip_util
 import six
 
-import pip.util
 import pkg_resources
 
 
@@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs):
 def build_rpm(options, filename, build_options):
     if os.path.isfile(filename):
         temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-')
-        pip.util.unpack_file(filename, temp_dir, None, None)
+        pip_util.unpack_file(filename, temp_dir, None, None)
         source_dir = temp_dir
         archive_name = filename
     elif os.path.isdir(filename):
@@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options):
     pkg_key = python_name_to_key(pkg_name)
     build_dir = options.rpm_base
     rpm_name = python_key_to_rpm(pkg_key)
-    version = one_line(info["version"])
+
+    # NOTE(harlowja): try not to use info["version"] to get the version, since
+    # currently that is getting normalized by setuptools to be a normalized
+    # version which can be different from the actual version...
+    try:
+        version = setup_py_one_line(source_dir, "--version")
+    except Exception:
+        version = info['version']
+        logger.warning("Failed extracting version, falling back to '%s'",
+                       version, exc_info=True)
     cleaned_version = version.replace('-', '_')
     spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)