openstack/rally-openstack
Code Issues Proposed changes

[requirements] Add helper for managing requirements

Browse Source 
While we are not depend on OpenStack global-requirements, we need to be synced
with it to be sure that Rally compatible with all OpenStack related stuff.

This patch provides new helper script, which checks versions from g-r, correct
license and latest version from pypi.

The usage is simple:

   # sync rally requirements with g-r
   tox -e requirements

   # just format rally requirements
   tox -e requirements -- --format

    # sync rally requirements with g-r & add upper limits for all packages
   tox -e requirements -- --add-uppers

Also, this patch formats Rally requirements in unified form, puts right
licenses to requirements and adds rally licenses in correct way in setup.cfg

Change-Id: Ia50694f636d9f80f08d90cc8133ebac8bb3f8671
This commit is contained in:
Andrey Kurilin 2016-07-20 15:54:39 +03:00
parent f0adca5c4b
commit 991d5b4abb
6 changed files with 377 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
requirements.txt
View File

@ -1,40 +1,44 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
alembic>=0.8.4 # MIT
boto>=2.32.1 # MIT
decorator>=3.4.0 # BSD
Jinja2>=2.8 # BSD License (3 clause)
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
netaddr!=0.7.16,>=0.7.12 # BSD
oslo.config>=3.10.0 # Apache-2.0
oslo.db>=4.1.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
oslo.serialization>=1.10.0 # Apache-2.0
oslo.utils>=3.11.0 # Apache-2.0
paramiko>=2.0 # LGPL
pbr>=1.6 # Apache-2.0
PrettyTable<0.8,>=0.7 # BSD
PyYAML>=3.1.0 # MIT
python-designateclient>=1.5.0 # Apache-2.0
python-glanceclient>=2.0.0 # Apache-2.0
python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache-2.0
keystoneauth1>=2.1.0 # Apache-2.0
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
python-neutronclient>=4.2.0 # Apache-2.0
python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache-2.0
python-manilaclient>=1.3.0 # Apache-2.0
python-heatclient>=1.1.0 # Apache-2.0
python-ceilometerclient>=2.2.1 # Apache-2.0
python-ironicclient>=1.1.0 # Apache-2.0
python-saharaclient>=0.13.0 # Apache-2.0
python-troveclient>=2.2.0 # Apache-2.0
python-zaqarclient>=1.0.0 # Apache-2.0
python-swiftclient>=2.2.0 # Apache-2.0
python-watcherclient>=0.23.0 # Apache-2.0
python-subunit>=0.0.18 # Apache-2.0/BSD
requests>=2.10.0 # Apache-2.0
SQLAlchemy<1.1.0,>=1.0.10 # MIT
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
six>=1.9.0 # MIT
alembic>=0.8.4 # MIT
boto>=2.32.1 # MIT
decorator>=3.4.0 # new BSD License
Jinja2>=2.8 # BSD
# [constant-upper-limit]
jsonschema!=2.5.0,>=2.0.0,<3.0.0 # MIT
netaddr!=0.7.16,>=0.7.12 # BSD
oslo.config>=3.10.0 # Apache Software License
oslo.db>=4.1.0 # Apache Software License
oslo.i18n>=2.1.0 # Apache Software License
oslo.log>=1.14.0 # Apache Software License
oslo.serialization>=1.10.0 # Apache Software License
oslo.utils>=3.11.0 # Apache Software License
paramiko>=2.0 # LGPL
pbr>=1.6 # Apache Software License
# [constant-upper-limit]
PrettyTable>=0.7,<0.8 # BSD
PyYAML>=3.1.0 # MIT
python-designateclient>=1.5.0 # Apache License, Version 2.0
python-glanceclient>=2.0.0 # Apache License, Version 2.0
python-keystoneclient!=1.8.0,!=2.1.0,>=1.7.0 # Apache Software License
keystoneauth1>=2.1.0 # Apache Software License
python-novaclient!=2.33.0,>=2.29.0 # Apache License, Version 2.0
python-neutronclient>=4.2.0 # Apache Software License
python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0 # Apache Software License
python-manilaclient>=1.3.0 # Apache Software License
python-heatclient>=1.1.0 # Apache Software License
python-ceilometerclient>=2.2.1 # Apache Software License
python-ironicclient>=1.1.0 # Apache Software License
python-saharaclient>=0.13.0 # Apache License, Version 2.0
python-troveclient>=2.2.0 # Apache Software License
python-zaqarclient>=1.0.0 # Apache Software License
python-swiftclient>=2.2.0 # Apache Software License
python-watcherclient>=0.23.0 # Apache Software License
python-subunit>=0.0.18
requests>=2.10.0 # Apache License, Version 2.0
# [constant-upper-limit]
SQLAlchemy>=1.0.10,<1.1.0 # MIT
# [constant-upper-limit]
sphinx!=1.2.0,!=1.3b1,>=1.1.2,<1.3 # BSD
six>=1.9.0 # MIT

1
setup.cfg
View File

@ -6,6 +6,7 @@ description-file =
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
license = Apache License, Version 2.0
classifier =
Environment :: OpenStack
Intended Audience :: Developers

23
test-requirements.txt
View File

@ -1,16 +1,17 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
hacking<0.10,>=0.9.2
pytest>=2.7
pytest-cov>=2.2.1
pytest-html
# [constant-upper-limit]
hacking>=0.9.2,<0.10 # Apache Software License
pytest>=2.7 # MIT license
pytest-cov>=2.2.1 # MIT
pytest-html # Mozilla Public License 2.0 (MPL 2.0)
coverage>=3.6 # Apache-2.0
ddt>=1.0.1 # MIT
mock>=2.0 # BSD
python-dateutil>=2.4.2 # BSD
testtools>=1.4.0 # MIT
coverage>=3.6 # Apache License, Version 2.0
ddt>=1.0.1
mock>=2.0
python-dateutil>=2.4.2 # Simplified BSD
testtools>=1.4.0
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
oslosphinx!=3.4.0,>=2.5.0 # Apache Software License
oslotest>=1.10.0 # Apache Software License

317
tests/ci/sync_requirements.py Normal file
View File

@ -0,0 +1,317 @@
#!/usr/bin/env python
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Synchronizes, formats and prepares requirements to release(obtains and adds
maximum allowed version).
"""
import argparse
import logging
import re
import sys
import textwrap
import requests
LOG = logging.getLogger(__name__)
if not LOG.handlers:
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)
GLOBAL_REQUIREMENTS_LOCATIONS = (
"https://raw.githubusercontent.com/openstack/requirements/master/",
"http://git.openstack.org/cgit/openstack/requirements/plain/"
)
GLOBAL_REQUIREMENTS_FILENAME = "global-requirements.txt"
RALLY_REQUIREMENTS_FILES = (
"requirements.txt",
"test-requirements.txt",
"optional-requirements.txt"
)
UPPER_LIMIT_TAG = "[constant-upper-limit]"
class Comment(object):
def __init__(self, s=None):
self._comments = []
self.is_finished = False
if s:
self.append(s)
def finish_him(self):
self.is_finished = True
def append(self, s):
self._comments.append(s[1:].strip())
def __str__(self):
return textwrap.fill("\n".join(self._comments), width=80,
initial_indent="# ", subsequent_indent="# ")
class Requirement(object):
RE_NAME = re.compile(r"[a-zA-Z0-9-._]+")
RE_CONST_VERSION = re.compile(r"==[a-zA-Z0-9.]+")
RE_MIN_VERSION = re.compile(r">=?[a-zA-Z0-9.]+")
RE_MAX_VERSION = re.compile(r"<=?[a-zA-Z0-9.]+")
RE_NE_VERSIONS = re.compile(r"!=[a-zA-Z0-9.]+")
# NOTE(andreykurilin): one license can have different labels. Let's use
# unified variant.
LICENSE_MAP = {"MIT license": "MIT",
"MIT License": "MIT",
"BSD License": "BSD",
"Apache 2.0": "Apache License, Version 2.0"}
def __init__(self, package_name, version):
self.package_name = package_name
self.version = version
self._license = None
self._pypy_info = None
self.upper_limit_flag = False
def sync_max_version_with_pypy(self):
if isinstance(self.version, dict) and not self.upper_limit_flag:
self.version["max"] = "<=%s" % self.pypy_info["info"]["version"]
@property
def pypy_info(self):
if self._pypy_info is None:
resp = requests.get("https://pypi.python.org/pypi/%s/json" %
self.package_name)
if resp.status_code != 200:
raise Exception(resp.text)
self._pypy_info = resp.json()
return self._pypy_info
@property
def license(self):
if self._license is None:
if self.pypy_info["info"]["license"]:
self._license = self.pypy_info["info"]["license"]
else:
# try to parse classifiers
prefix = "License :: OSI Approved :: "
classifiers = [c[len(prefix):]
for c in self.pypy_info["info"]["classifiers"]
if c.startswith(prefix)]
self._license = "/".join(classifiers)
self._license = self.LICENSE_MAP.get(self._license, self._license)
if self._license == "UNKNOWN":
self._license = None
return self._license
@classmethod
def parse_line(cls, line):
match = cls.RE_NAME.match(line)
if match:
name = match.group()
# remove name
versions = line.replace(name, "")
# remove comments
versions = versions.split("#")[0]
# remove python classifiers
versions = versions.split(";")[0].strip()
if not cls.RE_CONST_VERSION.match(versions):
versions = versions.strip().split(",")
min_version = None
max_version = None
ne_versions = []
for version in versions:
if cls.RE_MIN_VERSION.match(version):
if min_version:
raise Exception("Found several min versions for "
"%s package." % name)
min_version = version
elif cls.RE_MAX_VERSION.match(version):
if max_version:
raise Exception("Found several max versions for "
"%s package." % name)
max_version = version
elif cls.RE_NE_VERSIONS.match(version):
ne_versions.append(version)
versions = {"min": min_version,
"max": max_version,
"ne": ne_versions}
return cls(name, versions)
def __str__(self):
if isinstance(self.version, dict):
version = ""
if self.version["ne"]:
version += ",".join(self.version["ne"])
if self.version["min"]:
if version:
version += ","
version += self.version["min"]
if self.version["max"]:
if version:
version += ","
version += self.version["max"]
else:
version = self.version
string = "%s%s" % (self.package_name, version)
if self.license:
# NOTE(andreykurilin): When I start implementation of this script,
# python-keystoneclient dependency took around ~45-55, so let's
# use this length as indent. Feel free to modify it to lower or
# greater value.
magic_number = 55
if len(string) < magic_number:
indent = magic_number - len(string)
else:
indent = 2
string += " " * indent + "# " + self.license
return string
def __eq__(self, other):
return (isinstance(other, self.__class__) and
self.package_name == other.package_name)
def parse_data(raw_data, include_comments=True):
# first elem is None to simplify checks of last elem in requirements
requirements = [None]
for line in raw_data.split("\n"):
if line.startswith("#"):
if not include_comments:
continue
if (isinstance(requirements[-1], Comment) and
UPPER_LIMIT_TAG in line):
requirements[-1].is_finished = True
if getattr(requirements[-1], "is_finished", True):
requirements.append(Comment())
requirements[-1].append(line)
else:
if (isinstance(requirements[-1], Comment) and
not requirements[-1].is_finished):
requirements[-1].finish_him()
if line == "":
requirements.append("")
else:
# parse_line
req = Requirement.parse_line(line)
if req:
if (isinstance(requirements[-1], Comment) and
str(requirements[-1]).endswith(UPPER_LIMIT_TAG)):
req.upper_limit_flag = True
requirements.append(req)
if not requirements[-1]:
requirements.pop()
return requirements[1:]
def _read_requirements():
"""Read all rally requirements."""
LOG.info("Reading rally requirements...")
for file_name in RALLY_REQUIREMENTS_FILES:
LOG.debug("Try to read '%s'." % file_name)
with open(file_name) as f:
data = f.read()
LOG.info("Parsing requirements from %s." % file_name)
yield file_name, parse_data(data)
def _write_requirements(filename, requirements):
"""Saves requirements to file."""
LOG.info("Saving requirements to %s." % filename)
with open(filename, "w") as f:
for entity in requirements:
f.write(str(entity))
f.write("\n")
def _sync():
LOG.info("Obtaining global-requirements...")
for i in range(0, len(GLOBAL_REQUIREMENTS_LOCATIONS)):
url = GLOBAL_REQUIREMENTS_LOCATIONS[i] + GLOBAL_REQUIREMENTS_FILENAME
LOG.debug("Try to obtain global-requirements from %s" % url)
try:
raw_gr = requests.get(url).text
except requests.ConnectionError as e:
LOG.exception(e)
if i == len(GLOBAL_REQUIREMENTS_LOCATIONS) - 1:
# there are no more urls to try
raise Exception("Unable to obtain %s" %
GLOBAL_REQUIREMENTS_FILENAME)
else:
break
LOG.info("Parsing global-requirements...")
# NOTE(andreykurilin): global-requirements includes comments which can be
# unrelated to Rally project.
gr = parse_data(raw_gr, include_comments=False)
for filename, requirements in _read_requirements():
for i in range(0, len(requirements)):
if isinstance(requirements[i], Requirement):
try:
gr_item = gr[gr.index(requirements[i])]
except ValueError:
# it not g-r requirements
pass
else:
requirements[i].version = gr_item.version
yield filename, requirements
def sync():
"""Synchronizes Rally requirements with OpenStack global-requirements."""
for filename, requirements in _sync():
_write_requirements(filename, requirements)
def format_requirements():
"""Obtain package licenses from pypy and write requirements to file."""
for filename, requirements in _read_requirements():
_write_requirements(filename, requirements)
def add_uppers():
"""Obtains latest version of packages and put them to requirements."""
for filename, requirements in _sync():
LOG.info("Obtaining latest versions of packages from %s." % filename)
for req in requirements:
if isinstance(req, Requirement):
req.sync_max_version_with_pypy()
_write_requirements(filename, requirements)
def main():
parser = argparse.ArgumentParser(
prog="Python Requirement Manager for Rally",
description=__doc__.strip(),
add_help=True
)
action_groups = parser.add_mutually_exclusive_group()
action_groups.add_argument("--format",
action="store_const",
const=format_requirements,
dest="action")
action_groups.add_argument("--add-upper",
action="store_const",
const=add_uppers,
dest="action")
action_groups.set_defaults(action=sync)
parser.parse_args(sys.argv[1:]).action()
if __name__ == "__main__":
main()

3
tests/hacking/checks.py
View File

@ -154,7 +154,8 @@ def check_import_of_logging(logical_line, physical_line, filename):
excluded_files = ["./rally/common/logging.py",
"./tests/unit/test_logging.py",
"./tests/ci/rally_verify.py"]
"./tests/ci/rally_verify.py",
"./tests/ci/sync_requirements.py"]
forbidden_imports = ["from oslo_log",
"import oslo_log",

4
tox.ini
View File

@ -63,6 +63,10 @@ commands =
commands =
oslo-config-generator --config-file etc/rally/rally-config-generator.conf
[testenv:requirements]
deps = requests[security]
commands = python {toxinidir}/tests/ci/sync_requirements.py {posargs}
[flake8]
ignore = H703
show-source = true