From bcc1779defadf9e5cebf8e6959f746365c41a1eb Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 12 May 2016 16:15:46 -0400 Subject: [PATCH] add a validation command to ensure projects use check-requirements Add a command and tox environment to let us define a job to ensure that all repos in projects.txt have the check-requirements job defined. Two projects are missing the job now, so this patch depends on the patch to add those jobs. Change-Id: I61658a7265a77e72401978dda558f41b28a81d92 Depends-On: I81e1c70257e28583e776171580d75341e620bf8d Signed-off-by: Doug Hellmann --- .../cmds/validate_projects.py | 60 +++++++++++++++ openstack_requirements/project_config.py | 73 +++++++++++++++++++ setup.cfg | 1 + test-requirements.txt | 1 + tox.ini | 3 + 5 files changed, 138 insertions(+) create mode 100644 openstack_requirements/cmds/validate_projects.py create mode 100644 openstack_requirements/project_config.py diff --git a/openstack_requirements/cmds/validate_projects.py b/openstack_requirements/cmds/validate_projects.py new file mode 100644 index 0000000000..a880edc7cf --- /dev/null +++ b/openstack_requirements/cmds/validate_projects.py @@ -0,0 +1,60 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Apply validation rules to the projects.txt file + +""" + +import argparse + +from openstack_requirements import project_config + + +_BLACKLIST = set([ + # NOTE(dhellmann): It's not clear why these don't get updates, + # except that trying to do so may break the test jobs using them + # because of the nature of the projects. + 'openstack-dev/hacking', + 'openstack-dev/pbr', + # We can't enforce the check rules against this repo. + 'openstack/requirements', +]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'projects_list', + default='projects.txt', + help='path to the projects.txt file', + ) + args = parser.parse_args() + + zuul_layout = project_config.get_zuul_layout_data() + + error_count = 0 + + print('\nChecking %s' % args.projects_list) + with open(args.projects_list, 'r') as f: + for repo in f: + repo = repo.strip() + if repo.startswith('#'): + continue + if repo in _BLACKLIST: + continue + pe = project_config.require_check_requirements_for_repo( + zuul_layout, repo) + for e in pe: + print(e) + error_count += 1 + + return 1 if error_count else 0 diff --git a/openstack_requirements/project_config.py b/openstack_requirements/project_config.py new file mode 100644 index 0000000000..8c2fac528d --- /dev/null +++ b/openstack_requirements/project_config.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Work with the project-config repository. +""" + +import requests +import yaml + + +ZUUL_LAYOUT_URL = 'http://git.openstack.org/cgit/openstack-infra/project-config/plain/zuul/layout.yaml' # noqa +ZUUL_LAYOUT_FILENAME = 'openstack-infra/project-config/zuul/layout.yaml' + +# We use this key to modify the data structure read from the zuul +# layout file. We don't control what are valid keys there, so make it +# easy to change the key we use, just in case. +_VALIDATE_KEY = 'validate-projects-by-name' + + +def get_zuul_layout_data(url=ZUUL_LAYOUT_URL): + """Return the parsed data structure for the zuul/layout.yaml file. + + :param url: Optional URL to the location of the file. Defaults to + the most current version in the public git repository. + + """ + r = requests.get(url) + raw = yaml.safe_load(r.text) + # Add a mapping from repo name to repo settings, since that is how + # we access this most often. + raw[_VALIDATE_KEY] = { + p['name']: p + for p in raw['projects'] + } + return raw + + +def require_check_requirements_for_repo(zuul_layout, repo): + """Check the repository for the jobs related to requirements. + + Returns a list of error messages. + + """ + errors = [] + + if repo not in zuul_layout[_VALIDATE_KEY]: + errors.append( + ('did not find %s in %s' % (repo, ZUUL_LAYOUT_FILENAME), + True) + ) + else: + p = zuul_layout[_VALIDATE_KEY][repo] + templates = [ + t['name'] + for t in p.get('template', []) + ] + # NOTE(dhellmann): We don't mess around looking for individual + # jobs, because we want projects to use the templates. + if 'check-requirements' not in templates: + errors.append( + '%s no check-requirements job specified for %s' + % (ZUUL_LAYOUT_FILENAME, repo) + ) + return errors diff --git a/setup.cfg b/setup.cfg index 49f6a121f0..9533c88c31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,3 +34,4 @@ console_scripts = generate-constraints = openstack_requirements.cmds.generate:main update-requirements = openstack_requirements.cmds.update:main validate-constraints = openstack_requirements.cmds.validate:main + validate-projects = openstack_requirements.cmds.validate_projects:main diff --git a/test-requirements.txt b/test-requirements.txt index e87c04e837..c3fae1f7f6 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -14,3 +14,4 @@ setuptools>=16.0 # PSF/ZPL # this is required for the docs build jobs sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 +PyYAML>=3.1.0 diff --git a/tox.ini b/tox.ini index c2afb392a6..231642b737 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,9 @@ commands = generate-constraints {posargs} [testenv:validate] commands = validate-constraints {toxinidir}/global-requirements.txt {toxinidir}/upper-constraints.txt {toxinidir}/blacklist.txt +[testenv:validate-projects] +commands = validate-projects {toxinidir}/projects.txt + [testenv:pep8] commands = flake8