openstack/requirements
Code Issues Proposed changes

rearrange requirements check code to add tests

Browse Source 
Rearrange and refactor the existing requirements check logic to better
support testing.  Add a few tests for the function that determines if
a setting matches the global requirements.

Change-Id: I3ea9049cf728b11ef71ca180304ee7041303a409
Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann 2018-03-27 11:58:08 -04:00
parent f49bec99bc
commit 3a378a6c57
3 changed files with 232 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
openstack_requirements/check.py Normal file
View File

@ -0,0 +1,149 @@
# Copyright (C) 2011 OpenStack, LLC.
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
from openstack_requirements import project
from openstack_requirements import requirement
class RequirementsList(object):
def __init__(self, name, project):
self.name = name
self.reqs_by_file = {}
self.project = project
self.failed = False
@property
def reqs(self):
return {k: v for d in self.reqs_by_file.values()
for k, v in d.items()}
def extract_reqs(self, content, strict):
reqs = collections.defaultdict(set)
parsed = requirement.parse(content)
for name, entries in parsed.items():
if not name:
# Comments and other unprocessed lines
continue
list_reqs = [r for (r, line) in entries]
# Strip the comments out before checking if there are duplicates
list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
if strict and len(list_reqs_stripped) != len(set(
list_reqs_stripped)):
print("Requirements file has duplicate entries "
"for package %s : %r." % (name, list_reqs))
self.failed = True
reqs[name].update(list_reqs)
return reqs
def process(self, strict=True):
"""Convert the project into ready to use data.
- an iterable of requirement sets to check
- each set has the following rules:
- each has a list of Requirements objects
- duplicates are not permitted within that list
"""
print("Checking %(name)s" % {'name': self.name})
# First, parse.
for fname, content in self.project.get('requirements', {}).items():
print("Processing %(fname)s" % {'fname': fname})
if strict and not content.endswith('\n'):
print("Requirements file %s does not "
"end with a newline." % fname)
self.reqs_by_file[fname] = self.extract_reqs(content, strict)
for name, content in project.extras(self.project).items():
print("Processing .[%(extra)s]" % {'extra': name})
self.reqs_by_file[name] = self.extract_reqs(content, strict)
def _is_requirement_in_global_reqs(req, global_reqs):
# Compare all fields except the extras field as the global
# requirements should not have any lines with the extras syntax
# example: oslo.db[xyz]<1.2.3
for req2 in global_reqs:
if (req.package == req2.package and
req.location == req2.location and
req.specifiers == req2.specifiers and
req.markers == req2.markers and
req.comment == req2.comment):
return True
return False
def get_global_reqs(content):
"""Return global_reqs structure.
Parse content and return dict mapping names to sets of Requirement
objects."
"""
global_reqs = {}
parsed = requirement.parse(content)
for k, entries in parsed.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
return global_reqs
def validate(head_reqs, branch_reqs, blacklist, global_reqs):
failed = False
# iterate through the changing entries and see if they match the global
# equivalents we want enforced
for fname, freqs in head_reqs.reqs_by_file.items():
print("Validating %(fname)s" % {'fname': fname})
for name, reqs in freqs.items():
counts = {}
if (name in branch_reqs.reqs and
reqs == branch_reqs.reqs[name]):
# Unchanged [or a change that preserves a current value]
continue
if name in blacklist:
# Blacklisted items are not synced and are managed
# by project teams as they see fit, so no further
# testing is needed.
continue
if name not in global_reqs:
failed = True
print("Requirement %s not in openstack/requirements" %
str(reqs))
continue
if reqs == global_reqs[name]:
continue
for req in reqs:
if req.extras:
for extra in req.extras:
counts[extra] = counts.get(extra, 0) + 1
else:
counts[''] = counts.get('', 0) + 1
if not _is_requirement_in_global_reqs(
req, global_reqs[name]):
failed = True
print("Requirement for package %s : %s does "
"not match openstack/requirements value : %s" % (
name, str(req), str(global_reqs[name])))
for extra, count in counts.items():
if count != len(global_reqs[name]):
failed = True
print("Package %s%s requirement does not match "
"number of lines (%d) in "
"openstack/requirements" % (
name,
('[%s]' % extra) if extra else '',
len(global_reqs[name])))
return failed

75
openstack_requirements/tests/test_check.py Normal file
View File

@ -0,0 +1,75 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from openstack_requirements import check
from openstack_requirements import requirement
import fixtures
import testtools
class TestIsReqInGlobalReqs(testtools.TestCase):
def setUp(self):
super(TestIsReqInGlobalReqs, self).setUp()
self._stdout_fixture = fixtures.StringStream('stdout')
self.stdout = self.useFixture(self._stdout_fixture).stream
self.useFixture(fixtures.MonkeyPatch('sys.stdout', self.stdout))
self.global_reqs = check.get_global_reqs('name>=1.2,!=1.4')
print('global_reqs', self.global_reqs)
def test_match(self):
req = requirement.parse('name>=1.2,!=1.4')['name'][0][0]
self.assertTrue(
check._is_requirement_in_global_reqs(
req,
self.global_reqs['name'],
)
)
def test_name_mismatch(self):
req = requirement.parse('wrongname>=1.2,!=1.4')['wrongname'][0][0]
self.assertFalse(
check._is_requirement_in_global_reqs(
req,
self.global_reqs['name'],
)
)
def test_min_mismatch(self):
req = requirement.parse('name>=1.3,!=1.4')['name'][0][0]
self.assertFalse(
check._is_requirement_in_global_reqs(
req,
self.global_reqs['name'],
)
)
def test_extra_exclusion(self):
req = requirement.parse('name>=1.2,!=1.4,!=1.5')['name'][0][0]
self.assertFalse(
check._is_requirement_in_global_reqs(
req,
self.global_reqs['name'],
)
)
def test_missing_exclusion(self):
req = requirement.parse('name>=1.2')['name'][0][0]
self.assertFalse(
check._is_requirement_in_global_reqs(
req,
self.global_reqs['name'],
)
)

112
playbooks/files/project-requirements-change.py
View File

@ -16,7 +16,6 @@
# under the License.
import argparse
import collections
import contextlib
import os
import shlex
@ -28,6 +27,7 @@ import tempfile
requirement = None
project = None
check = None
def run_command(cmd):
@ -41,58 +41,6 @@ def run_command(cmd):
return (out.strip(), err.strip())
class RequirementsList(object):
def __init__(self, name, project):
self.name = name
self.reqs_by_file = {}
self.project = project
self.failed = False
@property
def reqs(self):
return {k: v for d in self.reqs_by_file.values()
for k, v in d.items()}
def extract_reqs(self, content, strict):
reqs = collections.defaultdict(set)
parsed = requirement.parse(content)
for name, entries in parsed.items():
if not name:
# Comments and other unprocessed lines
continue
list_reqs = [r for (r, line) in entries]
# Strip the comments out before checking if there are duplicates
list_reqs_stripped = [r._replace(comment='') for r in list_reqs]
if strict and len(list_reqs_stripped) != len(set(
list_reqs_stripped)):
print("Requirements file has duplicate entries "
"for package %s : %r." % (name, list_reqs))
self.failed = True
reqs[name].update(list_reqs)
return reqs
def process(self, strict=True):
"""Convert the project into ready to use data.
- an iterable of requirement sets to check
- each set has the following rules:
- each has a list of Requirements objects
- duplicates are not permitted within that list
"""
print("Checking %(name)s" % {'name': self.name})
# First, parse.
for fname, content in self.project.get('requirements', {}).items():
print("Processing %(fname)s" % {'fname': fname})
if strict and not content.endswith('\n'):
print("Requirements file %s does not "
"end with a newline." % fname)
self.reqs_by_file[fname] = self.extract_reqs(content, strict)
for name, content in project.extras(self.project).items():
print("Processing .[%(extra)s]" % {'extra': name})
self.reqs_by_file[name] = self.extract_reqs(content, strict)
def grab_args():
"""Grab and return arguments"""
parser = argparse.ArgumentParser(
@ -129,8 +77,10 @@ def install_and_load_requirements(reqroot, reqdir):
out, err = run_command("virtualenv " + req_venv)
out, err = run_command(req_pip + " install " + reqdir)
sys.path.append(req_lib)
global check
global project
global requirement
from openstack_requirements import check # noqa
from openstack_requirements import project # noqa
from openstack_requirements import requirement # noqa
@ -153,7 +103,6 @@ def main():
args = grab_args()
branch = args.branch
os.chdir(args.src_dir)
failed = False
reqdir = args.reqs
# build a list of requirements from the global list in the
@ -161,18 +110,15 @@ def main():
with tempdir() as reqroot:
install_and_load_requirements(reqroot, reqdir)
global_reqs = requirement.parse(
open(reqdir + '/global-requirements.txt', 'rt').read())
for k, entries in global_reqs.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
with open(reqdir + '/global-requirements.txt', 'rt') as f:
global_reqs = check.get_global_reqs(f.read())
blacklist = requirement.parse(
open(reqdir + '/blacklist.txt', 'rt').read())
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
head_proj = project.read(cwd)
head_reqs = RequirementsList('HEAD', head_proj)
head_reqs = check.RequirementsList('HEAD', head_proj)
# Don't apply strict parsing rules to stable branches.
# Reasoning is:
# - devstack etc protect us from functional issues
@ -194,53 +140,11 @@ def main():
run_command("git checkout %s" % branch)
else:
branch_proj = {'root': cwd}
branch_reqs = RequirementsList(branch, branch_proj)
branch_reqs = check.RequirementsList(branch, branch_proj)
# Don't error on the target branch being broken.
branch_reqs.process(strict=False)
# iterate through the changing entries and see if they match the global
# equivalents we want enforced
for fname, freqs in head_reqs.reqs_by_file.items():
print("Validating %(fname)s" % {'fname': fname})
for name, reqs in freqs.items():
counts = {}
if (name in branch_reqs.reqs and
reqs == branch_reqs.reqs[name]):
# Unchanged [or a change that preserves a current value]
continue
if name in blacklist:
# Blacklisted items are not synced and are managed
# by project teams as they see fit, so no further
# testing is needed.
continue
if name not in global_reqs:
failed = True
print("Requirement %s not in openstack/requirements" %
str(reqs))
continue
if reqs == global_reqs[name]:
continue
for req in reqs:
if req.extras:
for extra in req.extras:
counts[extra] = counts.get(extra, 0) + 1
else:
counts[''] = counts.get('', 0) + 1
if not _is_requirement_in_global_reqs(
req, global_reqs[name]):
failed = True
print("Requirement for package %s : %s does "
"not match openstack/requirements value : %s" % (
name, str(req), str(global_reqs[name])))
for extra, count in counts.items():
if count != len(global_reqs[name]):
failed = True
print("Package %s%s requirement does not match "
"number of lines (%d) in "
"openstack/requirements" % (
name,
('[%s]' % extra) if extra else '',
len(global_reqs[name])))
failed = check.validate(head_reqs, branch_reqs, blacklist, global_reqs)
# report the results
if failed or head_reqs.failed or branch_reqs.failed: