From 4bb92e041cb3f77bae04e53ab89be5fcc4529647 Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Thu, 23 Jun 2016 16:49:16 +1000 Subject: [PATCH] Integrate normalize as part of the requirements tools Rather than have this as an external tool that open codes some of the logic we have in the existing codebase we should integrate it. Change-Id: I979dcefeec4ae28705088d4fac1e41b4d33e40e0 --- cmds/normalize | 83 ------------------- .../cmds/normalize_requirements.py | 37 +++++++++ openstack_requirements/requirement.py | 33 ++++++-- setup.cfg | 1 + 4 files changed, 66 insertions(+), 88 deletions(-) delete mode 100755 cmds/normalize create mode 100644 openstack_requirements/cmds/normalize_requirements.py diff --git a/cmds/normalize b/cmds/normalize deleted file mode 100755 index 40dd6c48d0..0000000000 --- a/cmds/normalize +++ /dev/null @@ -1,83 +0,0 @@ -#! /usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import argparse -from distutils.version import LooseVersion - -import pkg_resources - - -def cmp_specifier(a, b): - weight = {'>=': 0, '>': 0, - '==': 1 , '~=': 1, '!=': 1, - '<': 2., '<=': 2} - wa, wb = weight[a[0]], weight[b[0]] - res = cmp(wa, wb) - if res != 0: - return res - else: - return cmp(LooseVersion(a[1]), LooseVersion(b[1])) - - -def lint(requirements): - output = [] - for line in requirements: - # comments and empty lines are untouched - if line.startswith("#") or line == "": - output.append(line) - continue - - # split comments - parts = line.split('#', 1) - if len(parts) > 1: - base, comments = parts - else: - base, comments = parts[0], "" - base, comments = base.strip(), comments.strip() - # split extras specifiers - parts = base.split(';', 1) - if len(parts) > 1: - base, extras = parts - else: - base, extras = parts[0], "" - base, extras = base.strip(), extras.strip() - - req = pkg_resources.Requirement.parse(base) - name = req.key - # FIXME: not py3 compliant - specs = ["%s%s" % x for x in sorted(req.specs, cmp=cmp_specifier)] - name += ','.join(specs) - if extras != "": - name += ";%s" % extras - if comments != "": - name += " #%s" % comments - output.append(name) - return output - - -def main(): - parser = argparse.ArgumentParser(description="Normalize requirements files") - parser.add_argument('requirements', - help=['requirements file input']) - args = parser.parse_args() - with open(args.requirements) as f: - requirements = [line.strip() for line in f.readlines()] - - for line in lint(requirements): - print(line) - - -if __name__ == '__main__': - main() diff --git a/openstack_requirements/cmds/normalize_requirements.py b/openstack_requirements/cmds/normalize_requirements.py new file mode 100644 index 0000000000..dad7721285 --- /dev/null +++ b/openstack_requirements/cmds/normalize_requirements.py @@ -0,0 +1,37 @@ +#! /usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +from openstack_requirements import requirement + + +def main(): + parser = argparse.ArgumentParser( + description="Normalize requirements files") + parser.add_argument('requirements', help='requirements file input') + args = parser.parse_args() + with open(args.requirements) as f: + requirements = [line.strip() for line in f.readlines()] + + for line in requirements: + req = requirement.parse_line(line) + print(req.to_line(comment_prefix=' ', + sort_specifiers=True), end='') + + +if __name__ == '__main__': + main() diff --git a/openstack_requirements/requirement.py b/openstack_requirements/requirement.py index 48ece717ba..18a9e5db74 100644 --- a/openstack_requirements/requirement.py +++ b/openstack_requirements/requirement.py @@ -15,9 +15,10 @@ # This module has no IO at all, and none should be added. import collections -import re - +import distutils.version +import packaging.specifiers import pkg_resources +import re # A header for the requirements file(s). @@ -31,6 +32,21 @@ _REQS_HEADER = [ ] +def cmp_specifier(a, b): + weight = {'>=': 0, '>': 0, + '==': 1, '~=': 1, '!=': 1, + '<': 2, '<=': 2} + a = a._spec + b = b._spec + wa, wb = weight[a[0]], weight[b[0]] + res = cmp(wa, wb) + if res != 0: + return res + else: + return cmp(distutils.version.LooseVersion(a[1]), + distutils.version.LooseVersion(b[1])) + + class Requirement(collections.namedtuple('Requirement', ['package', 'location', 'specifiers', 'markers', 'comment', 'extras'])): @@ -40,17 +56,24 @@ class Requirement(collections.namedtuple('Requirement', cls, package, location, specifiers, markers, comment, frozenset(extras or ())) - def to_line(self, marker_sep=';', line_prefix=''): - comment_p = ' ' if self.package else '' + def to_line(self, marker_sep=';', line_prefix='', comment_prefix=' ', + sort_specifiers=False): + comment_p = comment_prefix if self.package else '' comment = (comment_p + self.comment if self.comment else '') marker = marker_sep + self.markers if self.markers else '' package = line_prefix + self.package if self.package else '' location = self.location + '#egg=' if self.location else '' extras = '[%s]' % ",".join(sorted(self.extras)) if self.extras else '' + specifiers = self.specifiers + if sort_specifiers: + _specifiers = packaging.specifiers.SpecifierSet(specifiers) + _specifiers = ['%s' % s for s in sorted(_specifiers, + cmp=cmp_specifier)] + specifiers = ','.join(_specifiers) return '%s%s%s%s%s%s\n' % (location, package, extras, - self.specifiers, + specifiers, marker, comment) diff --git a/setup.cfg b/setup.cfg index e1c809662b..5cbb3270be 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,3 +36,4 @@ console_scripts = update-requirements = openstack_requirements.cmds.update:main validate-constraints = openstack_requirements.cmds.validate:main validate-projects = openstack_requirements.cmds.validate_projects:main + normalize-requirements = openstack_requirements.cmds.normalize_requirements:main