Add upper-constraints.txt generator.

This script when run creates a constraints file with exact pins for
the transitive dependencies of a requirements file.

Change-Id: I1282f8e4010c0ec46c473495bacddf044d06c0af

README.rst

@@ -73,7 +73,7 @@ However due to the interactions with transitive dependencies this doesn't
 actually deliver what we need.
 
 We are moving to a system where we will define the precise versions of all
-dependencies using ``upper-contraints.txt``. This will be overlaid onto all
+dependencies using ``upper-constraints.txt``. This will be overlaid onto all
 pip commands made during devstack, and by tox, and will provide a single,
 atomic, source of truth for what works at any given time. The constraints will
 be required to be compatible with the global requirements, and will

openstack_requirements/generate.py

@@ -0,0 +1,160 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import optparse
+import os.path
+import subprocess
+import sys
+
+import fixtures
+
+
+def _parse_freeze(text):
+    """Parse a freeze into structured data.
+
+    :param text: The output from a pip freeze command.
+    :return: A list of (package, version) tuples.
+    """
+    result = []
+    for line in text.splitlines():
+        line = line.strip()
+        if line.startswith('-'):
+            raise Exception("Irregular line: %s" % line)
+        if line.startswith('#'):
+            continue
+        if not line:
+            continue
+        package, version = line.split('==')[:2]
+        result.append((package, version))
+    return result
+
+
+def _freeze(requirements, python):
+    """Generate a frozen install from requirements.
+
+    A constraints file is the result of installing a set of requirements and
+    then freezing the result. We currently special case pip and setuptools
+    as pip does, excluding them from the set. We may however want to revisit
+    this in future if releases of those things break our gate.
+
+    In principle we should determine this by introspecting all the packages
+    transitively, since we need to deal wit environment markers....
+    but thats reimplementing a large chunk of pip (and since pip doesn't
+    resolve yet, differently too). For now, we take a list of Python
+    executables to test under, and then union the results. This is in fact the
+    key difference between a constraints file and a requirements file: we're
+    not triggering installation, so we can and will list packages that are
+    not relevant to e.g. Python3.4 in the constraints output.
+
+    :param requirements: The path to a requirements file to use when generating
+        the constraints.
+    :param python: A Python binary to use. E.g. /usr/bin/python3.4
+    :return: A tuple (python_version, list of (package, version)'s)
+    """
+    output = []
+    try:
+        version_out = subprocess.check_output(
+            [python, "--version"], stderr=subprocess.STDOUT)
+        output.append(version_out)
+        version_all = version_out.split()[1]
+        version = '.'.join(version_all.split('.')[:2])
+        with fixtures.TempDir() as temp:
+            output.append(subprocess.check_output(
+                ['virtualenv', '-p', python, temp.path]))
+            pip_bin = os.path.join(temp.path, 'bin', 'pip')
+            output.append(subprocess.check_output(
+                [pip_bin, 'install', '-U', 'pip', 'setuptools', 'wheel']))
+            output.append(subprocess.check_output(
+                [pip_bin, 'install', '-r', requirements]))
+            freeze = subprocess.check_output([pip_bin, 'freeze'])
+            output.append(freeze)
+            return (version, _parse_freeze(freeze))
+    except Exception as exc:
+        if isinstance(exc, subprocess.CalledProcessError):
+            output.append(exc.output)
+        raise Exception(
+            "Failed to generate freeze: %s %s"
+            % ('\n'.join(output), exc))
+
+
+def _combine_freezes(freezes):
+    """Combine multiple freezes into a single structure.
+
+    This deals with the variation between different python versions by
+    generating environment markers when different pythons need different
+    versions of a dependency.
+
+    :param freezes: A list of (python_version, frozen_requirements) tuples.
+    :return: A list of '\n' terminated lines for a requirements file.
+    """
+    packages = {}  # {package : {version : [py_version]}}
+    reference_versions = []
+    for py_version, freeze in freezes:
+        if py_version in reference_versions:
+            raise Exception("Duplicate python %s" % py_version)
+        reference_versions.append(py_version)
+        for package, version in freeze:
+            packages.setdefault(
+                package, {}).setdefault(version, []).append(py_version)
+
+    for package, versions in sorted(packages.items()):
+        if len(versions) != 1 or versions.values()[0] != reference_versions:
+            # markers
+            for version, py_versions in sorted(versions.items()):
+                # Once the ecosystem matures, we can consider using OR.
+                for py_version in sorted(py_versions):
+                    yield (
+                        "%s===%s;python_version=='%s'\n" %
+                        (package, version, py_version))
+        else:
+            # no markers
+            yield '%s===%s\n' % (package, versions.keys()[0])
+
+
+# -- untested UI glue from here down.
+
+def _validate_options(options):
+    """Check that options are valid.
+
+    :param options: The optparse options for this program.
+    """
+    if not options.pythons:
+        raise Exception("No Pythons given - see -p.")
+    for python in options.pythons:
+        if not os.path.exists(python):
+            raise Exception(
+                "Python %(python)s not found." % dict(python=python))
+    if not options.requirements:
+        raise Exception("No requirements file specified - see -r.")
+    if not os.path.exists(options.requirements):
+        raise Exception(
+            "Requirements file %(req)s not found."
+            % dict(req=options.requirementes))
+
+
+def main(argv=None, stdout=None):
+    parser = optparse.OptionParser()
+    parser.add_option(
+        "-p", dest="pythons", action="append",
+        help="Specify Python versions to use when generating constraints."
+             "e.g. -p /usr/bin/python3.4")
+    parser.add_option(
+        "-r", dest="requirements", help="Requirements file to process.")
+    options, args = parser.parse_args(argv)
+    if stdout is None:
+        stdout = sys.stdout
+    _validate_options(options)
+    freezes = [
+        _freeze(options.requirements, python) for python in options.pythons]
+    stdout.writelines(_combine_freezes(freezes))
+    stdout.flush()

openstack_requirements/tests/test_generate.py

@@ -0,0 +1,82 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os.path
+
+import fixtures
+import testtools
+from testtools import matchers
+
+from openstack_requirements import generate
+
+
+class TestFreeze(testtools.TestCase):
+
+    def test_freeze_smoke(self):
+        # Use an aribtrary python. The installation of virtualenv system wide
+        # is presumed.
+        versions = ['/usr/bin/python%(v)s' % dict(v=v) for v in
+                    ["2.7", "3.4"]]
+        found = [v for v in versions if os.path.exists(v)][0]
+        req = self.useFixture(fixtures.TempDir()).path + '/r.txt'
+        with open(req, 'wt') as output:
+            output.write('fixtures==1.2.0')
+        frozen = generate._freeze(req, found)
+        expected_version = found[-3:]
+        self.expectThat(frozen, matchers.HasLength(2))
+        self.expectThat(frozen[0], matchers.Equals(expected_version))
+        # There are multiple items in the dependency tree of fixtures.
+        # Since this is a smoke test, just ensure fixtures is there.
+        self.expectThat(frozen[1], matchers.Contains(('fixtures', '1.2.0')))
+
+
+class TestParse(testtools.TestCase):
+
+    def test_parse(self):
+        text = "linecache2==1.0.0\nargparse==1.2\n\n# fred\n"
+        parsed = generate._parse_freeze(text)
+        self.assertEqual(
+            [('linecache2', '1.0.0'), ('argparse', '1.2')], parsed)
+
+    def test_editable_banned(self):
+        text = "-e git:..."
+        self.assertRaises(Exception, generate._parse_freeze, text)  # noqa
+
+
+class TestCombine(testtools.TestCase):
+
+    def test_same_items(self):
+        fixtures = [('fixtures', '1.2.0')]
+        freeze_27 = ('2.7', fixtures)
+        freeze_34 = ('3.4', fixtures)
+        self.assertEqual(
+            ['fixtures===1.2.0\n'],
+            list(generate._combine_freezes([freeze_27, freeze_34])))
+
+    def test_distinct_items(self):
+        freeze_27 = ('2.7', [('fixtures', '1.2.0')])
+        freeze_34 = ('3.4', [('fixtures', '1.2.0'), ('enum', '1.5.0')])
+        self.assertEqual(
+            ["enum===1.5.0;python_version=='3.4'\n", 'fixtures===1.2.0\n'],
+            list(generate._combine_freezes([freeze_27, freeze_34])))
+
+    def test_different_versions(self):
+        freeze_27 = ('2.7', [('fixtures', '1.2.0')])
+        freeze_34 = ('3.4', [('fixtures', '1.5.0')])
+        self.assertEqual(
+            ["fixtures===1.2.0;python_version=='2.7'\n",
+             "fixtures===1.5.0;python_version=='3.4'\n"],
+            list(generate._combine_freezes([freeze_27, freeze_34])))
+
+    def test_duplicate_pythons(self):
+        with testtools.ExpectedException(Exception):
+            list(generate._combine_freezes([('2.7', []), ('2.7', [])]))

requirements.txt

@@ -1 +1,2 @@
+fixtures>=0.3.14
 Parsley

setup.cfg

@@ -23,4 +23,5 @@ packages =
 
 [entry_points]
 console_scripts =
+    generate-constraints = openstack_requirements.generate:main
     update-requirements = openstack_requirements.update:main

test-requirements.txt

@@ -1,8 +1,8 @@
 # NOTE: These are requirements for testing the requirements project only
 #       See global-requirements for the actual requirements list
 hacking>=0.10,<0.11
-fixtures>=0.3.14
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=0.9.36
 toposort>=1.0 # Apache 2.0
+virtualenv