diff --git a/requirements.txt b/requirements.txt index 7b0e75a1..5de7ff2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,50 +3,50 @@ # process, which may cause wedges in the gate later. # Rally core dependencies -alembic>=0.8.10,<=0.9.5 # MIT -decorator>=3.4.0,<=4.1.2 # new BSD License -Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4,<=2.9.6 # BSD -jsonschema>=2.0.0,!=2.5.0,<3.0.0 # MIT -morph -netaddr>=0.7.13,!=0.7.16,<=0.7.19 # BSD -oslo.config>=4.0.0,!=4.3.0,!=4.4.0,<=4.12.0 # Apache Software License -oslo.db>=4.24.0,<=4.26.0 # Apache Software License -oslo.log==3.30.0 # Apache Software License -oslo.utils>=3.20.0,<=3.29.0 # Apache Software License -paramiko>=2.0.0,<=2.2.1 # LGPL -pbr>=2.0.0,!=2.1.0,<=3.1.1 # Apache Software License +alembic>=0.8.10 # MIT +decorator>=3.4.0 # new BSD License +Jinja2>=2.8,!=2.9.0,!=2.9.1,!=2.9.2,!=2.9.3,!=2.9.4 # BSD +jsonschema>=2.6.0,<3.0.0 # MIT +morph # GPLv3+ +netaddr>=0.7.18 # BSD +oslo.config>=4.6.0 # Apache Software License +oslo.db>=4.27.0 # Apache Software License +oslo.log>=3.30.0 # Apache Software License +oslo.utils>=3.28.0 # Apache Software License +paramiko>=2.0.0 # LGPL +pbr>=2.0.0,!=2.1.0 # Apache Software License PrettyTable>=0.7.1,<0.8 # BSD -PyYAML>=3.10,<=3.12 # MIT -python-subunit>=0.0.18,<=1.2.0 -requests>=2.14.2,<=2.18.4 # Apache License, Version 2.0 -SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,<=1.2.0b2 # MIT -six>=1.9.0,<=1.10.0 # MIT -virtualenv>=13.1.0,<=15.1.0 # MIT +PyYAML>=3.10 # MIT +python-subunit>=0.0.18 # UNKNOWN +requests>=2.14.2 # Apache License, Version 2.0 +SQLAlchemy>=1.0.10,!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8 # MIT +six>=1.9.0 # MIT +virtualenv>=13.1.0 # MIT # OpenStack related -boto>=2.32.1,<=2.48.0 # MIT -gnocchiclient>=2.7.0,<=4.0.0 # Apache Software License -keystoneauth1==3.2.0 # Apache Software License -os-faults>=0.1.15,<0.2.0 # Apache Software License -osprofiler>=1.4.0,<=1.12.0 # Apache Software License -python-ceilometerclient>=2.5.0,<=2.9.0 # Apache Software License -python-cinderclient==3.2.0 # Apache Software License -python-designateclient>=1.5.0,<=2.7.0 # Apache License, Version 2.0 -python-heatclient>=1.6.1,<=1.11.0 # Apache Software License -python-glanceclient==2.8.0 # Apache License, Version 2.0 -python-ironicclient>=1.14.0,<=1.17.0 # Apache Software License -python-keystoneclient>=3.8.0,<=3.13.0 # Apache Software License -python-magnumclient>=2.0.0,<=2.7.0 # Apache Software License -python-manilaclient>=1.12.0,<=1.17.1 # Apache Software License -python-mistralclient>=3.1.0,<=3.1.2 # Apache Software License -python-muranoclient>=0.8.2,<=0.14.0 # Apache License, Version 2.0 -python-monascaclient==1.7.0 # Apache Software License -python-neutronclient>=6.3.0,<=6.5.0 # Apache Software License -python-novaclient==9.1.0 # Apache License, Version 2.0 -python-saharaclient>=1.1.0,<=1.3.0 # Apache License, Version 2.0 -python-senlinclient>=1.1.0,<=1.4.0 # Apache Software License -python-swiftclient>=3.2.0,<=3.4.0 # Apache Software License -python-troveclient>=2.2.0,<=2.12.0 # Apache Software License -python-watcherclient>=0.23.0,<=1.3.0 # Apache Software License -python-zaqarclient>=1.0.0,<=1.7.0 # Apache Software License -kubernetes>=1.0.0,<=3.0.0 # Apache License Version 2.0 +boto>=2.32.1 # MIT +gnocchiclient>=3.3.1 # Apache Software License +keystoneauth1>=3.2.0 # Apache Software License +os-faults>=0.1.15 # Apache Software License +osprofiler>=1.4.0 # Apache Software License +python-ceilometerclient>=2.5.0 # Apache Software License +python-cinderclient>=3.2.0 # Apache Software License +python-designateclient>=2.7.0 # Apache License, Version 2.0 +python-heatclient>=1.10.0 # Apache Software License +python-glanceclient>=2.8.0 # Apache License, Version 2.0 +python-ironicclient>=1.14.0 # Apache Software License +python-keystoneclient>=3.8.0 # Apache Software License +python-magnumclient>=2.0.0 # Apache Software License +python-manilaclient>=1.16.0 # Apache Software License +python-mistralclient>=3.1.0 # Apache Software License +python-muranoclient>=0.8.2 # Apache License, Version 2.0 +python-monascaclient>=1.7.0 # Apache Software License +python-neutronclient>=6.3.0 # Apache Software License +python-novaclient>=9.1.0 # Apache License, Version 2.0 +python-saharaclient>=1.2.0 # Apache License, Version 2.0 +python-senlinclient>=1.1.0 # Apache Software License +python-swiftclient>=3.2.0 # Apache Software License +python-troveclient>=2.2.0 # Apache Software License +python-watcherclient>=1.1.0 # Apache Software License +python-zaqarclient>=1.0.0 # Apache Software License +kubernetes>=1.0.0 # Apache License Version 2.0 diff --git a/test-requirements.txt b/test-requirements.txt index ed07a5fc..6e641781 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,25 +2,25 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=0.12.0,!=0.13.0,<=1.0.0 # Apache Software License +hacking>=0.12.0,!=0.13.0 # Apache Software License -pytest>=2.7,<=3.2.1 # MIT +pytest>=2.7 # MIT # py.test plugin for measuring coverage. -pytest-cov>=2.2.1,<=2.5.1 # MIT +pytest-cov>=2.2.1 # MIT # py.test plugin for generating HTML reports -pytest-html>=1.10.0,<=1.15.2 # Mozilla Public License 2.0 (MPL 2.0) +pytest-html>=1.10.0 # Mozilla Public License 2.0 (MPL 2.0) # py.test xdist plugin for distributed testing and loop-on-failing modes -pytest-xdist<=1.20.0 # MIT +pytest-xdist # MIT -coverage>=4.0,!=4.4,<=4.4.1 # Apache License, Version 2.0 -ddt>=1.0.1,<=1.1.1 -mock==2.0.0 -python-dateutil>=2.4.2,<=2.6.1 # Simplified BSD -testtools>=1.4.0,<=2.3.0 +coverage>=4.0,!=4.4 # Apache License, Version 2.0 +ddt>=1.0.1 # UNKNOWN +mock>=2.0.0 # UNKNOWN +python-dateutil>=2.4.2 # Simplified BSD +testtools>=1.4.0 # UNKNOWN -sphinx>=1.6.2,<=1.6.3 # BSD -oslosphinx>=4.7.0,<=4.15.1 # Apache Software License -oslotest>=1.10.0,<=2.17.0 # Apache Software License +sphinx>=1.6.2 # BSD +oslosphinx>=4.7.0 # Apache Software License +oslotest>=1.10.0 # Apache Software License -testresources>=0.2.4,<=2.0.1 -testscenarios>=0.4,<=0.5.0 +testresources>=2.0.0 # UNKNOWN +testscenarios>=0.4 # UNKNOWN diff --git a/tests/ci/sync_requirements.py b/tests/ci/sync_requirements.py index 641cebc9..343ea8ba 100644 --- a/tests/ci/sync_requirements.py +++ b/tests/ci/sync_requirements.py @@ -18,7 +18,7 @@ Synchronizes, formats and prepares requirements to release(obtains and adds maximum allowed version). """ -import argparse +import collections import logging import re import sys @@ -37,7 +37,6 @@ GLOBAL_REQUIREMENTS_LOCATIONS = ( "https://raw.githubusercontent.com/openstack/requirements/master/", "http://git.openstack.org/cgit/openstack/requirements/plain/" ) -GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt" RALLY_REQUIREMENTS_FILES = ( "requirements.txt", "test-requirements.txt" @@ -63,12 +62,10 @@ class Comment(object): initial_indent="# ", subsequent_indent="# ") -class Requirement(object): - RE_NAME = re.compile(r"[a-zA-Z0-9-._]+") - RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+") - RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+") - RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+") - RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+") +_PYPI_CACHE = {} + + +class PYPIPackage(object): # NOTE(andreykurilin): one license can have different labels. Let's use # unified variant. LICENSE_MAP = {"MIT license": "MIT", @@ -76,44 +73,73 @@ class Requirement(object): "BSD License": "BSD", "Apache 2.0": "Apache License, Version 2.0"} - def __init__(self, package_name, version): + def __init__(self, package_name): self.package_name = package_name - self.version = version - self._license = None - self._pypy_info = None - self.do_not_touch = False - - def sync_max_version_with_pypy(self): - if isinstance(self.version, dict) and not self.do_not_touch: - self.version["max"] = "<=%s" % self.pypy_info["info"]["version"] + self._pypi_info = None + self._pypi_license = None @property - def pypy_info(self): - if self._pypy_info is None: - resp = requests.get("https://pypi.python.org/pypi/%s/json" % - self.package_name) - if resp.status_code != 200: - raise Exception(resp.text) - self._pypy_info = resp.json() - return self._pypy_info + def pypi_info(self): + if self._pypi_info is None: + if self.package_name in _PYPI_CACHE: + self._pypi_info = _PYPI_CACHE[self.package_name] + else: + resp = requests.get("https://pypi.python.org/pypi/%s/json" % + self.package_name) + if resp.status_code != 200: + print("An error occurred while checking '%s' package at " + "pypi." % self.package_name) + raise Exception(resp.text) + self._pypi_info = resp.json() + + # let's cache it for the case when we need to sync requirements + # and update upper constrains + _PYPI_CACHE[self.package_name] = self._pypi_info + return self._pypi_info @property - def license(self): - if self._license is None: - if self.pypy_info["info"]["license"]: - self._license = self.pypy_info["info"]["license"] + def pypi_version(self): + return self.pypi_info["info"]["version"] + + @property + def pypi_license(self): + if self._pypi_license is None: + if self.pypi_info["info"]["license"]: + self._pypi_license = self.pypi_info["info"]["license"] else: # try to parse classifiers prefix = "License :: OSI Approved :: " classifiers = [c[len(prefix):] - for c in self.pypy_info["info"]["classifiers"] + for c in self.pypi_info["info"]["classifiers"] if c.startswith(prefix)] - self._license = "/".join(classifiers) - self._license = self.LICENSE_MAP.get(self._license, self._license) - if self._license == "UNKNOWN": - self._license = None + self._pypi_license = "/".join(classifiers) + self._license = self.LICENSE_MAP.get(self._pypi_license, + self._pypi_license) + if self._pypi_license == "UNKNOWN": + self._pypi_license = None return self._license + def __eq__(self, other): + return (isinstance(other, PYPIPackage) and + self.package_name == other.package_name) + + +class Requirement(PYPIPackage): + RE_NAME = re.compile(r"[a-zA-Z0-9-._]+") + RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+") + RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+") + RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+") + RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+") + + def __init__(self, package_name, version): + super(Requirement, self).__init__(package_name) + self.version = version + self.do_not_touch = False + + def sync_max_version_with_pypy(self): + if isinstance(self.version, dict) and not self.do_not_touch: + self.version["max"] = "<=%s" % self.pypi_version + @classmethod def parse_line(cls, line): match = cls.RE_NAME.match(line) @@ -180,7 +206,7 @@ class Requirement(object): version = ">=%s" % self.version[2:] string = "%s%s" % (self.package_name, version) - if self.license: + if self.pypi_license: # NOTE(andreykurilin): When I start implementation of this script, # python-keystoneclient dependency string took around ~45-55 # chars, so let's use this length as indent. Feel free to modify @@ -190,7 +216,7 @@ class Requirement(object): indent = magic_number - len(string) else: indent = 2 - string += " " * indent + "# " + self.license + string += " " * indent + "# " + self.pypi_license return string def __eq__(self, other): @@ -201,7 +227,35 @@ class Requirement(object): return not self.__eq__(other) -def parse_data(raw_data, include_comments=True): +class UpperConstraint(PYPIPackage): + + RE_LINE = re.compile( + r"(?P[a-zA-Z0-9-._]+)===(?P[a-zA-Z0-9.]+)") + + def __init__(self, package_name, version=None): + super(UpperConstraint, self).__init__(package_name) + self._version = version + + def __str__(self): + return "%s===%s" % (self.package_name, self.version) + + @property + def version(self): + if self._version is None: + self._version = self.pypi_version + return self._version + + @classmethod + def parse_line(cls, line): + match = cls.RE_LINE.match(line) + if match: + return cls(**match.groupdict()) + + def update(self, version): + self._version = version + + +def parse_data(raw_data, include_comments=True, dependency_cls=Requirement): # first elem is None to simplify checks of last elem in requirements requirements = [None] for line in raw_data.split("\n"): @@ -222,13 +276,15 @@ def parse_data(raw_data, include_comments=True): if (isinstance(requirements[-1], Comment) and not requirements[-1].is_finished): requirements[-1].finish_him() + # parse_line - req = Requirement.parse_line(line) - if req: + dep = dependency_cls.parse_line(line) + if dep: if (isinstance(requirements[-1], Comment) and DO_NOT_TOUCH_TAG in str(requirements[-1])): - req.do_not_touch = True - requirements.append(req) + dep.do_not_touch = True + requirements.append(dep) + for i in range(len(requirements) - 1, 0, -1): # remove empty lines at the end of file if isinstance(requirements[i], Comment): @@ -236,22 +292,27 @@ def parse_data(raw_data, include_comments=True): requirements.pop(i) else: break - return requirements[1:] + return collections.OrderedDict( + (v if isinstance(v, Comment) else v.package_name, v) + for v in requirements if v) -def _read_requirements(): - """Read all rally requirements.""" - LOG.info("Reading rally requirements...") - for file_name in RALLY_REQUIREMENTS_FILES: - LOG.debug("Try to read '%s'." % file_name) - with open(file_name) as f: - data = f.read() - LOG.info("Parsing requirements from %s." % file_name) - yield file_name, parse_data(data) +def _fetch_from_gr(filename): + """Try to fetch data from OpenStack global-requirements repo""" + for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)): + url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + filename + LOG.debug("Try to obtain %s from %s" % (filename, url)) + try: + return requests.get(url).text + except requests.ConnectionError as e: + LOG.exception(e) + raise Exception("Unable to obtain %s" % filename) def _write_requirements(filename, requirements): """Saves requirements to file.""" + if isinstance(requirements, dict): + requirements = requirements.values() LOG.info("Saving requirements to %s." % filename) with open(filename, "w") as f: for entity in requirements: @@ -259,82 +320,66 @@ def _write_requirements(filename, requirements): f.write("\n") -def _sync(): - LOG.info("Obtaining global-requirements...") - for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)): - url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME - LOG.debug("Try to obtain global-requirements from %s" % url) - try: - raw_gr = requests.get(url).text - except requests.ConnectionError as e: - LOG.exception(e) - if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1: - # there are no more urls to try - raise Exception("Unable to obtain %s" % - GLOBAL_REQUIREMENTS_FILENAME) - else: - break - - LOG.info("Parsing global-requirements...") - # NOTE(andreykurilin): global-requirements includes comments which can be - # unrelated to Rally project. - gr = parse_data(raw_gr, include_comments=False) - for filename, requirements in _read_requirements(): - for i in range(0, len(requirements)): - if (isinstance(requirements[i], Requirement) and - not requirements[i].do_not_touch): - try: - gr_item = gr[gr.index(requirements[i])] - except ValueError: - # it not g-r requirements - if isinstance(requirements[i].version, dict): - requirements[i].version["max"] = None - else: - requirements[i].version = gr_item.version - yield filename, requirements - - -def sync(): +def sync_requirements(): """Synchronizes Rally requirements with OpenStack global-requirements.""" - for filename, requirements in _sync(): - _write_requirements(filename, requirements) + LOG.info("Obtaining global-requirements of OpenStack...") + raw_gr = _fetch_from_gr("global-requirements.txt") + + # NOTE(andreykurilin): global-requirements includes comments which can be + # unrelated to Rally project, so let's just ignore them + gr = parse_data(raw_gr, include_comments=False) + for file_name in RALLY_REQUIREMENTS_FILES: + LOG.debug("Processing '%s'." % file_name) + with open(file_name) as f: + requirements = parse_data(f.read()) + for name, req in requirements.items(): + if isinstance(req, Requirement) and not req.do_not_touch: + if name in gr: + req.version = gr[req.package_name].version + else: + # it not g-r requirements + if isinstance(req.version, dict): + req.version["max"] = None + _write_requirements(file_name, requirements) -def format_requirements(): - """Obtain package licenses from pypy and write requirements to file.""" - for filename, requirements in _read_requirements(): - _write_requirements(filename, requirements) +def update_upper_constraints(): + """Obtains latest version of packages and put them to upper-constraints.""" + LOG.info("Obtaining upper-constrains from OpenStack...") + raw_g_uc = _fetch_from_gr("upper-constraints.txt") + # NOTE(andreykurilin): global OpenStack upper-constraints file includes + # comments which can be unrelated to Rally project, so let's just ignore + # them. + global_uc = parse_data(raw_g_uc, include_comments=False, + dependency_cls=UpperConstraint) + with open("upper-constraints.txt") as f: + our_uc = parse_data(f.read(), dependency_cls=UpperConstraint) + with open("requirements.txt") as f: + our_requirements = parse_data(f.read(), include_comments=False) + for name, req in our_requirements.items(): + if isinstance(req, Comment): + print("continue") + continue + print(req) + if name not in our_uc: + our_uc[name] = UpperConstraint(name) -def add_uppers(): - """Obtains latest version of packages and put them to requirements.""" - for filename, requirements in _sync(): - LOG.info("Obtaining latest versions of packages for %s." % filename) - for req in requirements: - if isinstance(req, Requirement): - if isinstance(req.version, dict) and not req.version["max"]: - req.sync_max_version_with_pypy() - _write_requirements(filename, requirements) + if name in global_uc: + # we cannot use whatever we want versions in CI. OpenStack CI + # installs ignores versions listed in requirements of + # particular project and use versions from global u-c file. + # It means that we need to suggest to use the same versions + our_uc[name].update(global_uc[name].version) + + our_uc = sorted(our_uc.values(), key=lambda o: o.package_name.upper()) + _write_requirements("upper-constraints.txt", our_uc) def main(): - parser = argparse.ArgumentParser( - prog="Python Requirement Manager for Rally", - description=__doc__.strip(), - add_help=True - ) + sync_requirements() + update_upper_constraints() - action_groups = parser.add_mutually_exclusive_group() - action_groups.add_argument("--format", - action="store_const", - const=format_requirements, - dest="action") - action_groups.add_argument("--add-upper", - action="store_const", - const=add_uppers, - dest="action") - action_groups.set_defaults(action=sync) - parser.parse_args(sys.argv[1:]).action() if __name__ == "__main__": sys.exit(main()) diff --git a/upper-constraints.txt b/upper-constraints.txt new file mode 100644 index 00000000..e791b22c --- /dev/null +++ b/upper-constraints.txt @@ -0,0 +1,45 @@ +alembic===0.9.6 +boto===2.48.0 +decorator===4.1.2 +gnocchiclient===4.0.0 +Jinja2===2.9.6 +jsonschema===2.6.0 +keystoneauth1===3.2.0 +kubernetes===2.0.0 +morph===0.1.2 +netaddr===0.7.19 +os-faults===0.1.17 +oslo.config===5.0.0 +oslo.db===4.29.0 +oslo.log===3.32.0 +oslo.utils===3.30.0 +osprofiler===1.13.0 +paramiko===2.3.1 +pbr===3.1.1 +PrettyTable===7 +python-ceilometerclient===2.9.0 +python-cinderclient===3.2.0 +python-designateclient===2.7.0 +python-glanceclient===2.8.0 +python-heatclient===1.12.0 +python-ironicclient===1.17.1 +python-keystoneclient===3.13.0 +python-magnumclient===2.7.0 +python-manilaclient===1.17.2 +python-mistralclient===3.1.3 +python-monascaclient===1.8.0 +python-muranoclient===0.14.0 +python-neutronclient===6.5.0 +python-novaclient===9.1.1 +python-saharaclient===1.3.0 +python-senlinclient===1.4.0 +python-subunit===1.2.0 +python-swiftclient===3.4.0 +python-troveclient===2.12.0 +python-watcherclient===1.4.0 +python-zaqarclient===1.7.0 +PyYAML===3.12 +requests===2.18.4 +six===1.11.0 +SQLAlchemy===1.1.14 +virtualenv===15.1.0