diff --git a/doc/source/usage.rst b/doc/source/usage.rst index 7aba41f..d16a2ee 100644 --- a/doc/source/usage.rst +++ b/doc/source/usage.rst @@ -44,6 +44,22 @@ Rendering the `example.spec.j2` file and also use the epochs can be done with:: The epoch file is optional. If a package name is not in the epochs file, epoch for that package is not used. +Handling requirements +********************* + +Updating versions for `Requires` and `BuildRequires` takes a lot of time. +:program:`renderspec` has the ability to insert versions from a given +`global-requirements.txt` file. The file must contain lines following `PEP0508`_ + +.. note:: For OpenStack, the `global-requirements.txt`_ can be used. + +To render a `example.spec.j2` file with a given requirements file, do:: + + renderspec --requirements global-requirements.txt example.spec.j2 + +.. _PEP0508: https://www.python.org/dev/peps/pep-0508/ +.. _global-requirements.txt: https://git.openstack.org/cgit/openstack/requirements/tree/global-requirements.txt + Template features ================= @@ -79,6 +95,11 @@ rendered on Fedora to:: BuildRequires: python-oslo-config >= 2:3.4.0 +It's also possible to skip adding required versions and handle that with a +`global-requirements.txt` file. Given that this file contains `oslo.config>=4.3.0` and +rendering with `--requirements`, the rendered spec would contain:: + + BuildRequires: python-oslo-config >= 4.3.0 context filter `license` ************************ diff --git a/renderspec/__init__.py b/renderspec/__init__.py index ce2f71e..5963383 100644 --- a/renderspec/__init__.py +++ b/renderspec/__init__.py @@ -29,17 +29,24 @@ import pymod2pkg import yaml +from renderspec import versions + def _context_epoch(context, pkg_name): """get the epoch (or 0 if unknown) for the given pkg name""" return context['epochs'].get(pkg_name, 0) -def _context_py2pkg(context, pkg_name, pkg_version=None): +def _context_py2pkg(context, pkg_name, pkg_version=None, old_filter=False): """generate a distro specific package name with optional version tuple.""" # package name handling name = pymod2pkg.module2package(pkg_name, context['spec_style']) + # if no pkg_version is given, look in the requirements and set one + if not pkg_version and not old_filter: + if pkg_name in context['requirements']: + pkg_version = ('>=', context['requirements'][pkg_name]) + # pkg_version is a tuple with comparator and number, i.e. "('>=', '1.2.3')" if pkg_version: # epoch handling @@ -103,7 +110,7 @@ def _filter_epoch(context, value): @contextfilter def _filter_python_module2package(context, pkg_name, pkg_version=None): - return _context_py2pkg(context, pkg_name, pkg_version) + return _context_py2pkg(context, pkg_name, pkg_version, old_filter=True) ################ @@ -134,7 +141,7 @@ def _env_register_filters_and_globals(env): env.globals['license'] = _globals_license_spdx -def generate_spec(spec_style, epochs, input_template_path): +def generate_spec(spec_style, epochs, requirements, input_template_path): """generate a spec file with the given style and the given template""" env = Environment(loader=FileSystemLoader( os.path.dirname(input_template_path))) @@ -142,7 +149,8 @@ def generate_spec(spec_style, epochs, input_template_path): _env_register_filters_and_globals(env) template = env.get_template(os.path.basename(input_template_path)) - return template.render(spec_style=spec_style, epochs=epochs) + return template.render(spec_style=spec_style, epochs=epochs, + requirements=requirements) def _get_default_distro(): @@ -178,6 +186,14 @@ def _get_epochs(filename): return {} +def _get_requirements(filename): + """get a dictionary with pkg-name->min-version mapping""" + if os.path.exists(filename): + with open(filename, 'r') as f: + return versions.get_requirements(f.readlines()) + return {} + + def process_args(): distro = _get_default_distro() parser = argparse.ArgumentParser( @@ -194,6 +210,11 @@ def process_args(): parser.add_argument("input-template", nargs='?', help="specfile jinja2 template to render. " "default: *.spec.j2") + parser.add_argument("--requirements", help="file which contains " + "PEP0508 compatible requirement lines." + "default: global-requirements.txt", + default="global-requirements.txt") + return vars(parser.parse_args()) @@ -216,7 +237,9 @@ def main(): output_fn, _, _ = input_template.rpartition('.') epochs = _get_epochs(args['epochs']) - spec = generate_spec(args['spec_style'], epochs, input_template) + requirements = _get_requirements(args['requirements']) + spec = generate_spec(args['spec_style'], epochs, requirements, + input_template) if output_fn and output_fn != '-': print("Rendering: %s -> %s" % (input_template, output_fn)) with open(output_fn, "w") as o: diff --git a/renderspec/versions.py b/renderspec/versions.py new file mode 100644 index 0000000..ee68cdc --- /dev/null +++ b/renderspec/versions.py @@ -0,0 +1,54 @@ +#!/usr/bin/python +# Copyright (c) 2016 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +from packaging.requirements import Requirement + + +def get_requirements(lines): + """parse the given lines and return a dict with pkg_name->version. + lines must follow PEP0508""" + requires = {} + for l in lines: + # skip comments and empty lines + if l.startswith('#') or len(l.strip()) == 0: + continue + # remove trailing comments + l = l.split('#')[0].rstrip(' ') + r = Requirement(l) + # check if we need the requirement + if r.marker: + # TODO (toabctl): currently we hardcode python 2.7 and linux2 + # see https://www.python.org/dev/peps/pep-0508/#environment-markers + marker_env = {'python_version': '2.7', 'sys_platform': 'linux'} + if not r.marker.evaluate(environment=marker_env): + continue + if r.specifier: + # we want the lowest possible version + # NOTE(toabctl): "min(r.specifier)" doesn't work. + # see https://github.com/pypa/packaging/issues/69 + lowest = None + for s in r.specifier: + # we don't want a lowest version which is not allowed + if s.operator == '!=': + continue + if not lowest or s.version < lowest.version: + lowest = s + + if lowest: + requires[r.name] = lowest.version + return requires diff --git a/requirements.txt b/requirements.txt index ea0303a..8502e2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Jinja2>=2.6 pymod2pkg>=0.3 PyYAML>=3.1.0 # MIT +packaging>=16.5 diff --git a/test-requirements.txt b/test-requirements.txt index 13057b8..ac8d484 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ flake8 testrepository>=0.0.18 testresources>=0.2.4 testtools>=1.4.0 +ddt sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 diff --git a/tests.py b/tests.py index 4ad1216..1e3811e 100644 --- a/tests.py +++ b/tests.py @@ -20,11 +20,15 @@ try: except ImportError: import unittest +from ddt import data, ddt, unpack + from jinja2 import Environment import renderspec +import renderspec.versions +@ddt class RenderspecContextFunctionTests(unittest.TestCase): """test functions which do some calculation based on the context""" def test_context_license_spdx(self): @@ -39,112 +43,88 @@ class RenderspecContextFunctionTests(unittest.TestCase): 'ASL 2.0' ) - def test_context_py2pkg_pgkname_only(self): + @data( + # without version + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + 'oslo.config', None, 'python-oslo.config'), + ({'spec_style': 'fedora', 'epochs': {}, 'requirements': {}}, + 'oslo.config', None, 'python-oslo-config'), + # with version + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + 'oslo.config', ('>=', '1.2.3'), 'python-oslo.config >= 1.2.3'), + ({'spec_style': 'fedora', 'epochs': {}, 'requirements': {}}, + 'oslo.config', ('==', '1.2.3~a0'), 'python-oslo-config == 1.2.3~a0'), + # with version, with epoch + ({'spec_style': 'suse', 'epochs': {'oslo.config': 4}, + 'requirements': {}}, + 'oslo.config', ('>=', '1.2.3'), 'python-oslo.config >= 4:1.2.3'), + # without version, with epoch + ({'spec_style': 'suse', 'epochs': {'oslo.config': 4}, + 'requirements': {}}, + 'oslo.config', None, 'python-oslo.config'), + # with version, with requirements + ({'spec_style': 'suse', 'epochs': {}, + 'requirements': {'oslo.config' '1.2.3'}}, + 'oslo.config', ('>=', '4.5.6'), 'python-oslo.config >= 4.5.6'), + # without version, with requirements + ({'spec_style': 'suse', 'epochs': {}, + 'requirements': {'oslo.config': '1.2.3'}}, + 'oslo.config', None, 'python-oslo.config >= 1.2.3'), + # without version, with requirements, with epoch + ({'spec_style': 'suse', 'epochs': {'oslo.config': 4}, + 'requirements': {'oslo.config': '1.2.3'}}, + 'oslo.config', None, 'python-oslo.config >= 4:1.2.3'), + # with version, with requirements, with epoch + ({'spec_style': 'suse', 'epochs': {'oslo.config': 4}, + 'requirements': {'oslo.config' '1.2.3'}}, + 'oslo.config', ('>=', '4.5.6'), 'python-oslo.config >= 4:4.5.6'), + ) + @unpack + def test_context_py2pkg(self, context, pkg_name, pkg_version, + expected_result): self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'suse', 'epochs': {}}, 'oslo.config'), - 'python-oslo.config' - ) - self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'fedora', 'epochs': {}}, 'oslo.config'), - 'python-oslo-config' - ) - - def test_context_py2pkg_pgkname_and_version(self): - self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'suse', 'epochs': {}}, - 'oslo.config', ('>=', '1.2.3')), - 'python-oslo.config >= 1.2.3' - ) - self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'fedora', 'epochs': {}}, - 'oslo.config', ('==', '1.2.3~a0')), - 'python-oslo-config == 1.2.3~a0' - ) - - def test_context_py2pkg_pgkname_and_version_and_epoch(self): - self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'suse', 'epochs': {'oslo.config': '4'}}, - 'oslo.config', ('>=', '1.2.3')), - 'python-oslo.config >= 4:1.2.3' - ) - - def test_context_py2pkg_pgkname_and_epoch_no_version(self): - self.assertEqual( - renderspec._context_py2pkg( - {'spec_style': 'suse', 'epochs': {'oslo.config': '4'}}, - 'oslo.config'), - 'python-oslo.config' - ) + renderspec._context_py2pkg(context, pkg_name, pkg_version), + expected_result) def test_context_epoch_without_epochs(self): self.assertEqual( renderspec._context_epoch( - {'spec_style': 'suse', 'epochs': {}}, 'oslo.config'), - 0 - ) + {'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + 'oslo.config'), 0) def test_context_epoch_with_epochs(self): self.assertEqual( renderspec._context_epoch( - {'spec_style': 'suse', 'epochs': {'oslo.config': 4}}, - 'oslo.config'), - 4 - ) + {'spec_style': 'suse', 'epochs': {'oslo.config': 4}, + 'requirements': {}}, 'oslo.config'), 4) +@ddt class RenderspecTemplateFilterTests(unittest.TestCase): def setUp(self): """create a Jinja2 environment and register the standard filters""" self.env = Environment() renderspec._env_register_filters_and_globals(self.env) - def test_render_filter_py2pkg_oldstyle(self): - template = self.env.from_string("{{ 'requests' | py2pkg }} >= 2.8.1") + @data( + # old style + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + "{{ 'requests' | py2pkg }}", "python-requests"), + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + "{{ 'requests' | py2pkg }} >= 2.8.1", "python-requests >= 2.8.1"), + ({'spec_style': 'suse', 'epochs': {}, + 'requirements': {'requests': '1.2.3'}}, + "{{ 'requests' | py2pkg }} >= 2.8.1", "python-requests >= 2.8.1"), + ) + @unpack + def test_render_filter_py2pkg(self, context, string, expected_result): + template = self.env.from_string(string) self.assertEqual( - template.render(spec_style='suse', epochs={}), - 'python-requests >= 2.8.1') - - def test_render_filter_py2pkg(self): - template = self.env.from_string( - "{{ 'requests' | py2pkg }}") - self.assertEqual( - template.render(spec_style='suse', epochs={}), - 'python-requests') - - def test_render_filter_py2pkg_with_version(self): - template = self.env.from_string( - "{{ 'requests' | py2pkg(('>=', '2.8.1')) }}") - self.assertEqual( - template.render(spec_style='suse', epochs={}), - 'python-requests >= 2.8.1') - - def test_render_filter_py2pkg_with_version_and_epoch(self): - template = self.env.from_string( - "{{ 'requests' | py2pkg(('>=', '2.8.1')) }}") - self.assertEqual( - template.render(spec_style='suse', epochs={'requests': '1'}), - 'python-requests >= 1:2.8.1') - - def test_render_filter_epoch_without_epochs(self): - template = self.env.from_string( - "{{ 'requests' | epoch }}") - self.assertEqual( - template.render(spec_style='suse', epochs={}), - '0') - - def test_render_filter_epoch_with_epochs(self): - template = self.env.from_string( - "{{ 'requests' | epoch }}") - self.assertEqual( - template.render(spec_style='suse', epochs={'requests': '1'}), - '1') + template.render(**context), + expected_result) +@ddt class RenderspecTemplateFunctionTests(unittest.TestCase): def setUp(self): """create a Jinja2 environment and register the standard filters""" @@ -155,44 +135,88 @@ class RenderspecTemplateFunctionTests(unittest.TestCase): template = self.env.from_string( "{{ license('Apache-2.0') }}") self.assertEqual( - template.render(spec_style='fedora', epochs={}), + template.render(spec_style='fedora', epochs={}, requirements={}), 'ASL 2.0') - def test_render_func_py2pkg(self): - template = self.env.from_string( - "{{ py2pkg('requests') }}") - self.assertEqual( - template.render(spec_style='suse', epochs={}), - 'python-requests') + @data( + # plain + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + "{{ py2pkg('requests') }}", "python-requests"), + # with version + ({'spec_style': 'suse', 'epochs': {}, 'requirements': {}}, + "{{ py2pkg('requests', ('>=', '2.8.1')) }}", + "python-requests >= 2.8.1"), + # with version, with epoch + ({'spec_style': 'suse', 'epochs': {'requests': 4}, 'requirements': {}}, + "{{ py2pkg('requests', ('>=', '2.8.1')) }}", + "python-requests >= 4:2.8.1"), + # with version, with epoch, with requirements + ({'spec_style': 'suse', 'epochs': {'requests': 4}, + 'requirements': {'requests': '1.2.3'}}, + "{{ py2pkg('requests', ('>=', '2.8.1')) }}", + "python-requests >= 4:2.8.1"), + # without version, with epoch, with requirements + ({'spec_style': 'suse', 'epochs': {'requests': 4}, + 'requirements': {'requests': '1.2.3'}}, + "{{ py2pkg('requests') }}", + "python-requests >= 4:1.2.3"), - def test_render_func_py2pkg_with_version(self): - template = self.env.from_string( - "{{ py2pkg('requests', ('>=', '2.8.1')) }}") + ) + @unpack + def test_render_func_py2pkg(self, context, string, expected_result): + template = self.env.from_string(string) self.assertEqual( - template.render(spec_style='suse', epochs={}), - 'python-requests >= 2.8.1') - - def test_render_func_py2pkg_with_version_and_epoch(self): - template = self.env.from_string( - "{{ py2pkg('requests', ('>=', '2.8.1')) }}") - self.assertEqual( - template.render(spec_style='suse', epochs={'requests': '1'}), - 'python-requests >= 1:2.8.1') + template.render(**context), + expected_result) def test_render_func_epoch_without_epochs(self): template = self.env.from_string( "Epoch: {{ epoch('requests') }}") self.assertEqual( - template.render(spec_style='suse', epochs={}), + template.render(spec_style='suse', epochs={}, requirements={}), 'Epoch: 0') def test_render_func_epoch_with_epochs(self): template = self.env.from_string( "Epoch: {{ epoch('requests') }}") self.assertEqual( - template.render(spec_style='suse', epochs={'requests': 1}), + template.render(spec_style='suse', epochs={'requests': 1}, + requirements={}), 'Epoch: 1') +class RenderspecVersionsTests(unittest.TestCase): + def test_without_version(self): + requires = renderspec.versions.get_requirements( + ['# a comment', '', ' ', 'pyasn1 # BSD', 'Paste']) + self.assertEqual(requires, {}) + + def test_with_single_version(self): + requires = renderspec.versions.get_requirements( + ['paramiko>=1.16.0 # LGPL']) + self.assertEqual(requires, {'paramiko': '1.16.0'}) + + def test_with_multiple_versions(self): + requires = renderspec.versions.get_requirements( + ['sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 # BSD']) + self.assertEqual(requires, {'sphinx': '1.1.2'}) + + def test_with_multiple_versions_and_invalid_lowest(self): + requires = renderspec.versions.get_requirements( + ['sphinx>=1.1.2,!=1.1.0,!=1.3b1,<1.3 # BSD']) + self.assertEqual(requires, {'sphinx': '1.1.2'}) + + def test_with_single_marker(self): + requires = renderspec.versions.get_requirements( + ["pywin32>=1.0;sys_platform=='win32' # PSF"]) + self.assertEqual(requires, {}) + + def test_with_multiple_markers(self): + requires = renderspec.versions.get_requirements( + ["""pyinotify>=0.9.6;sys_platform!='win32' and \ + sys_platform!='darwin' and sys_platform!='sunos5' # MIT"""]) + self.assertEqual(requires, {'pyinotify': '0.9.6'}) + + if __name__ == '__main__': unittest.main()