Merge "Stop enforcing py2 requirements match starting with U"

openstack_requirements/check.py

@@ -22,6 +22,8 @@ from packaging import specifiers
 from openstack_requirements import project
 from openstack_requirements import requirement
 
+MIN_PY_VERSION = '3.5'
+
 
 class RequirementsList(object):
     def __init__(self, name, project):
@@ -147,7 +149,22 @@ def get_global_reqs(content):
     return global_reqs
 
 
-def _validate_one(name, reqs, blacklist, global_reqs):
+def _get_python3_reqs(reqs):
+    """Filters out the reqs that are less than our minimum version."""
+    results = []
+    for req in reqs:
+        if not req.markers:
+            results.append(req)
+        else:
+            req_markers = markers.Marker(req.markers)
+            if req_markers.evaluate({
+                'python_version': MIN_PY_VERSION,
+            }):
+                results.append(req)
+    return results
+
+
+def _validate_one(name, reqs, blacklist, global_reqs, allow_3_only=False):
     """Returns True if there is a failure."""
     if name in blacklist:
         # Blacklisted items are not synced and are managed
@@ -174,7 +191,18 @@ def _validate_one(name, reqs, blacklist, global_reqs):
                   name)
             return True
     for extra, count in counts.items():
+        # Make sure the number of entries matches. If allow_3_only, then we
+        # just need to make sure we have at least the number of entries for
+        # supported Python 3 versions.
         if count != len(global_reqs[name]):
+            if (allow_3_only and
+                    count >= len(_get_python3_reqs(global_reqs[name]))):
+                print("WARNING: Package '%s%s' is only tracking python 3 "
+                      "requirements" % (
+                        name,
+                        ('[%s]' % extra) if extra else ''))
+                continue
+
             print("ERROR: Package '%s%s' requirement does not match "
                   "number of lines (%d) in "
                   "openstack/requirements" % (
@@ -185,7 +213,7 @@ def _validate_one(name, reqs, blacklist, global_reqs):
     return False
 
 
-def validate(head_reqs, blacklist, global_reqs):
+def validate(head_reqs, blacklist, global_reqs, allow_3_only=False):
     failed = False
     # iterate through the changing entries and see if they match the global
     # equivalents we want enforced
@@ -198,6 +226,7 @@ def validate(head_reqs, blacklist, global_reqs):
                     reqs,
                     blacklist,
                     global_reqs,
+                    allow_3_only,
                 )
                 or failed
             )

openstack_requirements/tests/test_check.py

@@ -331,6 +331,83 @@ class TestValidateOne(testtools.TestCase):
             )
         )
 
+    def test_new_item_matches_py3_allowed_no_version(self):
+        # If the global list has multiple entries for an item but the branch
+        # allows python 3 only, then only the py3 entries need to match.
+        # Requirements without a python_version marker should always be used.
+        r_content = textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        other-name
+        """)
+        reqs = [
+            r
+            for r, line in requirement.parse(r_content)['name']
+        ]
+        global_reqs = check.get_global_reqs(textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        name>=1.2,!=1.4;python_version=='2.6'
+        other-name
+        """))
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                blacklist=requirement.parse(''),
+                global_reqs=global_reqs,
+                allow_3_only=True,
+            )
+        )
+
+    def test_new_item_matches_py3_allowed_with_py2(self):
+        # If the global list has multiple entries for an item but the branch
+        # allows python 3 only, then only the py3 entries need to match.
+        # It should continue to pass with py2 entries though.
+        r_content = textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        name>=1.2,!=1.4;python_version=='2.6'
+        """)
+        reqs = [
+            r
+            for r, line in requirement.parse(r_content)['name']
+        ]
+        global_reqs = check.get_global_reqs(textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        name>=1.2,!=1.4;python_version=='2.6'
+        """))
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                blacklist=requirement.parse(''),
+                global_reqs=global_reqs,
+                allow_3_only=True,
+            )
+        )
+
+    def test_new_item_matches_py3_allowed_no_py2(self):
+        # If the global list has multiple entries for an item but the branch
+        # allows python 3 only, then only the py3 entries need to match.
+        r_content = textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        """)
+        reqs = [
+            r
+            for r, line in requirement.parse(r_content)['name']
+        ]
+        global_reqs = check.get_global_reqs(textwrap.dedent("""
+        name>=1.5;python_version=='3.5'
+        name>=1.2,!=1.4;python_version=='2.6'
+        """))
+        self.assertFalse(
+            check._validate_one(
+                'name',
+                reqs=reqs,
+                blacklist=requirement.parse(''),
+                global_reqs=global_reqs,
+                allow_3_only=True,
+            )
+        )
+
 
 class TestValidateLowerConstraints(testtools.TestCase):
 

playbooks/files/project-requirements-change.py

@@ -18,6 +18,7 @@
 import argparse
 import contextlib
 import os
+import re
 import shlex
 import shutil
 import subprocess
@@ -29,6 +30,9 @@ from openstack_requirements import project  # noqa
 from openstack_requirements import requirement  # noqa
 
 
+PYTHON_3_BRANCH = re.compile(r'^stable\/[u-z].*')
+
+
 def run_command(cmd):
     print(cmd)
     cmd_list = shlex.split(str(cmd))
@@ -124,8 +128,16 @@ def main():
         #    either.
         head_strict = not branch.startswith('stable/')
         head_reqs.process(strict=head_strict)
+        # Starting with Ussuri and later, we only need to be strict about
+        # Python 3 requirements.
+        python_3_branch = head_strict or PYTHON_3_BRANCH.match(branch)
 
-        failed = check.validate(head_reqs, blacklist, global_reqs)
+        failed = check.validate(
+            head_reqs,
+            blacklist,
+            global_reqs,
+            allow_3_only=python_3_branch,
+        )
 
         failed = (
             check.validate_lower_constraints(