From 991d5b4abbaa13d42dcc8e021d8cbab23626f4ed Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Wed, 20 Jul 2016 15:54:39 +0300 Subject: [PATCH] [requirements] Add helper for managing requirements While we are not depend on OpenStack global-requirements, we need to be synced with it to be sure that Rally compatible with all OpenStack related stuff. This patch provides new helper script, which checks versions from g-r, correct license and latest version from pypi. The usage is simple: # sync rally requirements with g-r tox -e requirements # just format rally requirements tox -e requirements -- --format # sync rally requirements with g-r & add upper limits for all packages tox -e requirements -- --add-uppers Also, this patch formats Rally requirements in unified form, puts right licenses to requirements and adds rally licenses in correct way in setup.cfg Change-Id: Ia50694f636d9f80f08d90cc8133ebac8bb3f8671 --- requirements.txt | 78 +++++---- setup.cfg | 1 + test-requirements.txt | 23 +-- tests/ci/sync_requirements.py | 317 ++++++++++++++++++++++++++++++++++ tests/hacking/checks.py | 3 +- tox.ini | 4 + 6 files changed, 377 insertions(+), 49 deletions(-) create mode 100644 tests/ci/sync_requirements.py diff --git a/requirements.txt b/requirements.txt index 997fe78b..36835406 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,44 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -alembic>=0.8.4 # MIT -boto>=2.32.1 # MIT -decorator>=3.4.0 # BSD -Jinja2>=2.8 # BSD License (3 clause) -jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT -netaddr!=0.7.16,>=0.7.12 # BSD -oslo.config>=3.10.0 # Apache-2.0 -oslo.db>=4.1.0 # Apache-2.0 -oslo.i18n>=2.1.0 # Apache-2.0 -oslo.log>=1.14.0 # Apache-2.0 -oslo.serialization>=1.10.0 # Apache-2.0 -oslo.utils>=3.11.0 # Apache-2.0 -paramiko>=2.0 # LGPL -pbr>=1.6 # Apache-2.0 -PrettyTable<0.8,>=0.7 # BSD -PyYAML>=3.1.0 # MIT -python-designateclient>=1.5.0 # Apache-2.0 -python-glanceclient>=2.0.0 # Apache-2.0 -python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache-2.0 -keystoneauth1>=2.1.0 # Apache-2.0 -python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0 -python-neutronclient>=4.2.0 # Apache-2.0 -python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0 -python-manilaclient>=1.3.0 # Apache-2.0 -python-heatclient>=1.1.0 # Apache-2.0 -python-ceilometerclient>=2.2.1 # Apache-2.0 -python-ironicclient>=1.1.0 # Apache-2.0 -python-saharaclient>=0.13.0 # Apache-2.0 -python-troveclient>=2.2.0 # Apache-2.0 -python-zaqarclient>=1.0.0 # Apache-2.0 -python-swiftclient>=2.2.0 # Apache-2.0 -python-watcherclient>=0.23.0 # Apache-2.0 -python-subunit>=0.0.18 # Apache-2.0/BSD -requests>=2.10.0 # Apache-2.0 -SQLAlchemy<1.1.0,>=1.0.10 # MIT -sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD -six>=1.9.0 # MIT +alembic>=0.8.4 # MIT +boto>=2.32.1 # MIT +decorator>=3.4.0 # new BSD License +Jinja2>=2.8 # BSD +# [constant-upper-limit] +jsonschema!=2.5.0,>=2.0.0,<3.0.0 # MIT +netaddr!=0.7.16,>=0.7.12 # BSD +oslo.config>=3.10.0 # Apache Software License +oslo.db>=4.1.0 # Apache Software License +oslo.i18n>=2.1.0 # Apache Software License +oslo.log>=1.14.0 # Apache Software License +oslo.serialization>=1.10.0 # Apache Software License +oslo.utils>=3.11.0 # Apache Software License +paramiko>=2.0 # LGPL +pbr>=1.6 # Apache Software License +# [constant-upper-limit] +PrettyTable>=0.7,<0.8 # BSD +PyYAML>=3.1.0 # MIT +python-designateclient>=1.5.0 # Apache License, Version 2.0 +python-glanceclient>=2.0.0 # Apache License, Version 2.0 +python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache Software License +keystoneauth1>=2.1.0 # Apache Software License +python-novaclient!=2.33.0,>=2.29.0 # Apache License, Version 2.0 +python-neutronclient>=4.2.0 # Apache Software License +python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache Software License +python-manilaclient>=1.3.0 # Apache Software License +python-heatclient>=1.1.0 # Apache Software License +python-ceilometerclient>=2.2.1 # Apache Software License +python-ironicclient>=1.1.0 # Apache Software License +python-saharaclient>=0.13.0 # Apache License, Version 2.0 +python-troveclient>=2.2.0 # Apache Software License +python-zaqarclient>=1.0.0 # Apache Software License +python-swiftclient>=2.2.0 # Apache Software License +python-watcherclient>=0.23.0 # Apache Software License +python-subunit>=0.0.18 +requests>=2.10.0 # Apache License, Version 2.0 +# [constant-upper-limit] +SQLAlchemy>=1.0.10,<1.1.0 # MIT +# [constant-upper-limit] +sphinx!=1.2.0,!=1.3b1,>=1.1.2,<1.3 # BSD +six>=1.9.0 # MIT diff --git a/setup.cfg b/setup.cfg index 11ab8a43..2d0c04c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,7 @@ description-file = author = OpenStack author-email = openstack-dev@lists.openstack.org home-page = http://www.openstack.org/ +license = Apache License, Version 2.0 classifier = Environment :: OpenStack Intended Audience :: Developers diff --git a/test-requirements.txt b/test-requirements.txt index f00a80bf..81cf0e7e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,16 +1,17 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking<0.10,>=0.9.2 -pytest>=2.7 -pytest-cov>=2.2.1 -pytest-html +# [constant-upper-limit] +hacking>=0.9.2,<0.10 # Apache Software License +pytest>=2.7 # MIT license +pytest-cov>=2.2.1 # MIT +pytest-html # Mozilla Public License 2.0 (MPL 2.0) -coverage>=3.6 # Apache-2.0 -ddt>=1.0.1 # MIT -mock>=2.0 # BSD -python-dateutil>=2.4.2 # BSD -testtools>=1.4.0 # MIT +coverage>=3.6 # Apache License, Version 2.0 +ddt>=1.0.1 +mock>=2.0 +python-dateutil>=2.4.2 # Simplified BSD +testtools>=1.4.0 -oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 -oslotest>=1.10.0 # Apache-2.0 +oslosphinx!=3.4.0,>=2.5.0 # Apache Software License +oslotest>=1.10.0 # Apache Software License diff --git a/tests/ci/sync_requirements.py b/tests/ci/sync_requirements.py new file mode 100644 index 00000000..23c158c2 --- /dev/null +++ b/tests/ci/sync_requirements.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Synchronizes, formats and prepares requirements to release(obtains and adds +maximum allowed version). +""" + +import argparse +import logging +import re +import sys +import textwrap + +import requests + + +LOG = logging.getLogger(__name__) +if not LOG.handlers: + LOG.addHandler(logging.StreamHandler()) + LOG.setLevel(logging.INFO) + + +GLOBAL_REQUIREMENTS_LOCATIONS = ( + "https://raw.githubusercontent.com/openstack/requirements/master/", + "http://git.openstack.org/cgit/openstack/requirements/plain/" +) +GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt" +RALLY_REQUIREMENTS_FILES = ( + "requirements.txt", + "test-requirements.txt", + "optional-requirements.txt" +) +UPPER_LIMIT_TAG = "[constant-upper-limit]" + + +class Comment(object): + def __init__(self, s=None): + self._comments = [] + self.is_finished = False + if s: + self.append(s) + + def finish_him(self): + self.is_finished = True + + def append(self, s): + self._comments.append(s[1:].strip()) + + def __str__(self): + return textwrap.fill("\n".join(self._comments), width=80, + initial_indent="# ", subsequent_indent="# ") + + +class Requirement(object): + RE_NAME = re.compile(r"[a-zA-Z0-9-._]+") + RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+") + RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+") + RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+") + RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+") + # NOTE(andreykurilin): one license can have different labels. Let's use + # unified variant. + LICENSE_MAP = {"MIT license": "MIT", + "MIT License": "MIT", + "BSD License": "BSD", + "Apache 2.0": "Apache License, Version 2.0"} + + def __init__(self, package_name, version): + self.package_name = package_name + self.version = version + self._license = None + self._pypy_info = None + self.upper_limit_flag = False + + def sync_max_version_with_pypy(self): + if isinstance(self.version, dict) and not self.upper_limit_flag: + self.version["max"] = "<=%s" % self.pypy_info["info"]["version"] + + @property + def pypy_info(self): + if self._pypy_info is None: + resp = requests.get("https://pypi.python.org/pypi/%s/json" % + self.package_name) + if resp.status_code != 200: + raise Exception(resp.text) + self._pypy_info = resp.json() + return self._pypy_info + + @property + def license(self): + if self._license is None: + if self.pypy_info["info"]["license"]: + self._license = self.pypy_info["info"]["license"] + else: + # try to parse classifiers + prefix = "License :: OSI Approved :: " + classifiers = [c[len(prefix):] + for c in self.pypy_info["info"]["classifiers"] + if c.startswith(prefix)] + self._license = "/".join(classifiers) + self._license = self.LICENSE_MAP.get(self._license, self._license) + if self._license == "UNKNOWN": + self._license = None + return self._license + + @classmethod + def parse_line(cls, line): + match = cls.RE_NAME.match(line) + if match: + name = match.group() + # remove name + versions = line.replace(name, "") + # remove comments + versions = versions.split("#")[0] + # remove python classifiers + versions = versions.split(";")[0].strip() + if not cls.RE_CONST_VERSION.match(versions): + versions = versions.strip().split(",") + min_version = None + max_version = None + ne_versions = [] + for version in versions: + if cls.RE_MIN_VERSION.match(version): + if min_version: + raise Exception("Found several min versions for " + "%s package." % name) + min_version = version + elif cls.RE_MAX_VERSION.match(version): + if max_version: + raise Exception("Found several max versions for " + "%s package." % name) + max_version = version + elif cls.RE_NE_VERSIONS.match(version): + ne_versions.append(version) + versions = {"min": min_version, + "max": max_version, + "ne": ne_versions} + return cls(name, versions) + + def __str__(self): + if isinstance(self.version, dict): + version = "" + if self.version["ne"]: + version += ",".join(self.version["ne"]) + if self.version["min"]: + if version: + version += "," + version += self.version["min"] + if self.version["max"]: + if version: + version += "," + version += self.version["max"] + else: + version = self.version + + string = "%s%s" % (self.package_name, version) + if self.license: + # NOTE(andreykurilin): When I start implementation of this script, + # python-keystoneclient dependency took around ~45-55, so let's + # use this length as indent. Feel free to modify it to lower or + # greater value. + magic_number = 55 + if len(string) < magic_number: + indent = magic_number - len(string) + else: + indent = 2 + string += " " * indent + "# " + self.license + return string + + def __eq__(self, other): + return (isinstance(other, self.__class__) and + self.package_name == other.package_name) + + +def parse_data(raw_data, include_comments=True): + # first elem is None to simplify checks of last elem in requirements + requirements = [None] + for line in raw_data.split("\n"): + if line.startswith("#"): + if not include_comments: + continue + + if (isinstance(requirements[-1], Comment) and + UPPER_LIMIT_TAG in line): + requirements[-1].is_finished = True + + if getattr(requirements[-1], "is_finished", True): + requirements.append(Comment()) + + requirements[-1].append(line) + else: + if (isinstance(requirements[-1], Comment) and + not requirements[-1].is_finished): + requirements[-1].finish_him() + if line == "": + requirements.append("") + else: + # parse_line + req = Requirement.parse_line(line) + if req: + if (isinstance(requirements[-1], Comment) and + str(requirements[-1]).endswith(UPPER_LIMIT_TAG)): + req.upper_limit_flag = True + requirements.append(req) + if not requirements[-1]: + requirements.pop() + return requirements[1:] + + +def _read_requirements(): + """Read all rally requirements.""" + LOG.info("Reading rally requirements...") + for file_name in RALLY_REQUIREMENTS_FILES: + LOG.debug("Try to read '%s'." % file_name) + with open(file_name) as f: + data = f.read() + LOG.info("Parsing requirements from %s." % file_name) + yield file_name, parse_data(data) + + +def _write_requirements(filename, requirements): + """Saves requirements to file.""" + LOG.info("Saving requirements to %s." % filename) + with open(filename, "w") as f: + for entity in requirements: + f.write(str(entity)) + f.write("\n") + + +def _sync(): + LOG.info("Obtaining global-requirements...") + for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)): + url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME + LOG.debug("Try to obtain global-requirements from %s" % url) + try: + raw_gr = requests.get(url).text + except requests.ConnectionError as e: + LOG.exception(e) + if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1: + # there are no more urls to try + raise Exception("Unable to obtain %s" % + GLOBAL_REQUIREMENTS_FILENAME) + else: + break + + LOG.info("Parsing global-requirements...") + # NOTE(andreykurilin): global-requirements includes comments which can be + # unrelated to Rally project. + gr = parse_data(raw_gr, include_comments=False) + for filename, requirements in _read_requirements(): + for i in range(0, len(requirements)): + if isinstance(requirements[i], Requirement): + try: + gr_item = gr[gr.index(requirements[i])] + except ValueError: + # it not g-r requirements + pass + else: + requirements[i].version = gr_item.version + yield filename, requirements + + +def sync(): + """Synchronizes Rally requirements with OpenStack global-requirements.""" + for filename, requirements in _sync(): + _write_requirements(filename, requirements) + + +def format_requirements(): + """Obtain package licenses from pypy and write requirements to file.""" + for filename, requirements in _read_requirements(): + _write_requirements(filename, requirements) + + +def add_uppers(): + """Obtains latest version of packages and put them to requirements.""" + for filename, requirements in _sync(): + LOG.info("Obtaining latest versions of packages from %s." % filename) + for req in requirements: + if isinstance(req, Requirement): + req.sync_max_version_with_pypy() + _write_requirements(filename, requirements) + + +def main(): + parser = argparse.ArgumentParser( + prog="Python Requirement Manager for Rally", + description=__doc__.strip(), + add_help=True + ) + + action_groups = parser.add_mutually_exclusive_group() + action_groups.add_argument("--format", + action="store_const", + const=format_requirements, + dest="action") + action_groups.add_argument("--add-upper", + action="store_const", + const=add_uppers, + dest="action") + action_groups.set_defaults(action=sync) + parser.parse_args(sys.argv[1:]).action() + +if __name__ == "__main__": + main() diff --git a/tests/hacking/checks.py b/tests/hacking/checks.py index c422175a..3d5fc9a0 100644 --- a/tests/hacking/checks.py +++ b/tests/hacking/checks.py @@ -154,7 +154,8 @@ def check_import_of_logging(logical_line, physical_line, filename): excluded_files = ["./rally/common/logging.py", "./tests/unit/test_logging.py", - "./tests/ci/rally_verify.py"] + "./tests/ci/rally_verify.py", + "./tests/ci/sync_requirements.py"] forbidden_imports = ["from oslo_log", "import oslo_log", diff --git a/tox.ini b/tox.ini index 3c354ee7..b7572542 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,10 @@ commands = commands = oslo-config-generator --config-file etc/rally/rally-config-generator.conf +[testenv:requirements] +deps = requests[security] +commands = python {toxinidir}/tests/ci/sync_requirements.py {posargs} + [flake8] ignore = H703 show-source = true