openstack/renderspec
Code Issues Proposed changes

Support package versions from global requirements file

Browse Source 
renderspec can now get a global-requirements.txt file to insert
versions into rendered spec files.
Example:

renderspec --requirements global-requirements.txt example.spec.j2

If a version is explicitly given via the py2pkg() method,
this version is used. If no version is given, renderspec tries
to find a version from the global-requirements.txt file.

Also update the documentation and for the new feature and simplify
the unittest by using ddt.

Change-Id: Ic0058f8aa9c6a1e7250e0e08f262bb3e2663d68b
This commit is contained in:
Thomas Bechtold 2016-03-09 09:43:58 +01:00
parent 049a9e6459
commit 5f46f91121
6 changed files with 236 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
doc/source/usage.rst
View File

@ -44,6 +44,22 @@ Rendering the `example.spec.j2` file and also use the epochs can be done with::
The epoch file is optional. If a package name is not in the epochs file,
epoch for that package is not used.
Handling requirements
*********************
Updating versions for `Requires` and `BuildRequires` takes a lot of time.
:program:`renderspec` has the ability to insert versions from a given
`global-requirements.txt` file. The file must contain lines following `PEP0508`_
.. note:: For OpenStack, the `global-requirements.txt`_ can be used.
To render a `example.spec.j2` file with a given requirements file, do::
renderspec --requirements global-requirements.txt example.spec.j2
.. _PEP0508: https://www.python.org/dev/peps/pep-0508/
.. _global-requirements.txt: https://git.openstack.org/cgit/openstack/requirements/tree/global-requirements.txt
Template features
=================
@ -79,6 +95,11 @@ rendered on Fedora to::
BuildRequires: python-oslo-config >= 2:3.4.0
It's also possible to skip adding required versions and handle that with a
`global-requirements.txt` file. Given that this file contains `oslo.config>=4.3.0` and
rendering with `--requirements`, the rendered spec would contain::
BuildRequires: python-oslo-config >= 4.3.0
context filter `license`
************************

33
renderspec/__init__.py
View File

@ -29,17 +29,24 @@ import pymod2pkg
import yaml
from renderspec import versions
def _context_epoch(context, pkg_name):
"""get the epoch (or 0 if unknown) for the given pkg name"""
return context['epochs'].get(pkg_name, 0)
def _context_py2pkg(context, pkg_name, pkg_version=None):
def _context_py2pkg(context, pkg_name, pkg_version=None, old_filter=False):
"""generate a distro specific package name with optional version tuple."""
# package name handling
name = pymod2pkg.module2package(pkg_name, context['spec_style'])
# if no pkg_version is given, look in the requirements and set one
if not pkg_version and not old_filter:
if pkg_name in context['requirements']:
pkg_version = ('>=', context['requirements'][pkg_name])
# pkg_version is a tuple with comparator and number, i.e. "('>=', '1.2.3')"
if pkg_version:
# epoch handling
@ -103,7 +110,7 @@ def _filter_epoch(context, value):
@contextfilter
def _filter_python_module2package(context, pkg_name, pkg_version=None):
return _context_py2pkg(context, pkg_name, pkg_version)
return _context_py2pkg(context, pkg_name, pkg_version, old_filter=True)
################
@ -134,7 +141,7 @@ def _env_register_filters_and_globals(env):
env.globals['license'] = _globals_license_spdx
def generate_spec(spec_style, epochs, input_template_path):
def generate_spec(spec_style, epochs, requirements, input_template_path):
"""generate a spec file with the given style and the given template"""
env = Environment(loader=FileSystemLoader(
os.path.dirname(input_template_path)))
@ -142,7 +149,8 @@ def generate_spec(spec_style, epochs, input_template_path):
_env_register_filters_and_globals(env)
template = env.get_template(os.path.basename(input_template_path))
return template.render(spec_style=spec_style, epochs=epochs)
return template.render(spec_style=spec_style, epochs=epochs,
requirements=requirements)
def _get_default_distro():
@ -178,6 +186,14 @@ def _get_epochs(filename):
return {}
def _get_requirements(filename):
"""get a dictionary with pkg-name->min-version mapping"""
if os.path.exists(filename):
with open(filename, 'r') as f:
return versions.get_requirements(f.readlines())
return {}
def process_args():
distro = _get_default_distro()
parser = argparse.ArgumentParser(
@ -194,6 +210,11 @@ def process_args():
parser.add_argument("input-template", nargs='?',
help="specfile jinja2 template to render. "
"default: *.spec.j2")
parser.add_argument("--requirements", help="file which contains "
"PEP0508 compatible requirement lines."
"default: global-requirements.txt",
default="global-requirements.txt")
return vars(parser.parse_args())
@ -216,7 +237,9 @@ def main():
output_fn, _, _ = input_template.rpartition('.')
epochs = _get_epochs(args['epochs'])
spec = generate_spec(args['spec_style'], epochs, input_template)
requirements = _get_requirements(args['requirements'])
spec = generate_spec(args['spec_style'], epochs, requirements,
input_template)
if output_fn and output_fn != '-':
print("Rendering: %s -> %s" % (input_template, output_fn))
with open(output_fn, "w") as o:

54
renderspec/versions.py Normal file
View File

@ -0,0 +1,54 @@
#!/usr/bin/python
# Copyright (c) 2016 SUSE Linux GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from packaging.requirements import Requirement
def get_requirements(lines):
"""parse the given lines and return a dict with pkg_name->version.
lines must follow PEP0508"""
requires = {}
for l in lines:
# skip comments and empty lines
if l.startswith('#') or len(l.strip()) == 0:
continue
# remove trailing comments
l = l.split('#')[0].rstrip(' ')
r = Requirement(l)
# check if we need the requirement
if r.marker:
# TODO (toabctl): currently we hardcode python 2.7 and linux2
# see https://www.python.org/dev/peps/pep-0508/#environment-markers
marker_env = {'python_version': '2.7', 'sys_platform': 'linux'}
if not r.marker.evaluate(environment=marker_env):
continue
if r.specifier:
# we want the lowest possible version
# NOTE(toabctl): "min(r.specifier)" doesn't work.
# see https://github.com/pypa/packaging/issues/69
lowest = None
for s in r.specifier:
# we don't want a lowest version which is not allowed
if s.operator == '!=':
continue
if not lowest or s.version < lowest.version:
lowest = s
if lowest:
requires[r.name] = lowest.version
return requires

1
requirements.txt
View File

@ -1,3 +1,4 @@
Jinja2>=2.6
pymod2pkg>=0.3
PyYAML>=3.1.0 # MIT
packaging>=16.5

1
test-requirements.txt
View File

@ -2,6 +2,7 @@ flake8
testrepository>=0.0.18
testresources>=0.2.4
testtools>=1.4.0
ddt
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0

238
tests.py
View File

@ -20,11 +20,15 @@ try:
except ImportError:
import unittest
from ddt import data, ddt, unpack
from jinja2 import Environment
import renderspec
import renderspec.versions
@ddt
class RenderspecContextFunctionTests(unittest.TestCase):
"""test functions which do some calculation based on the context"""
def test_context_license_spdx(self):
@ -39,112 +43,88 @@ class RenderspecContextFunctionTests(unittest.TestCase):
'ASL 2.0'
)
def test_context_py2pkg_pgkname_only(self):
@data(
# without version
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
'oslo.config', None, 'python-oslo.config'),
({'spec_style': 'fedora', 'epochs': {}, 'requirements': {}},
'oslo.config', None, 'python-oslo-config'),
# with version
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
'oslo.config', ('>=', '1.2.3'), 'python-oslo.config >= 1.2.3'),
({'spec_style': 'fedora', 'epochs': {}, 'requirements': {}},
'oslo.config', ('==', '1.2.3~a0'), 'python-oslo-config == 1.2.3~a0'),
# with version, with epoch
({'spec_style': 'suse', 'epochs': {'oslo.config': 4},
'requirements': {}},
'oslo.config', ('>=', '1.2.3'), 'python-oslo.config >= 4:1.2.3'),
# without version, with epoch
({'spec_style': 'suse', 'epochs': {'oslo.config': 4},
'requirements': {}},
'oslo.config', None, 'python-oslo.config'),
# with version, with requirements
({'spec_style': 'suse', 'epochs': {},
'requirements': {'oslo.config' '1.2.3'}},
'oslo.config', ('>=', '4.5.6'), 'python-oslo.config >= 4.5.6'),
# without version, with requirements
({'spec_style': 'suse', 'epochs': {},
'requirements': {'oslo.config': '1.2.3'}},
'oslo.config', None, 'python-oslo.config >= 1.2.3'),
# without version, with requirements, with epoch
({'spec_style': 'suse', 'epochs': {'oslo.config': 4},
'requirements': {'oslo.config': '1.2.3'}},
'oslo.config', None, 'python-oslo.config >= 4:1.2.3'),
# with version, with requirements, with epoch
({'spec_style': 'suse', 'epochs': {'oslo.config': 4},
'requirements': {'oslo.config' '1.2.3'}},
'oslo.config', ('>=', '4.5.6'), 'python-oslo.config >= 4:4.5.6'),
)
@unpack
def test_context_py2pkg(self, context, pkg_name, pkg_version,
expected_result):
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'suse', 'epochs': {}}, 'oslo.config'),
'python-oslo.config'
)
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'fedora', 'epochs': {}}, 'oslo.config'),
'python-oslo-config'
)
def test_context_py2pkg_pgkname_and_version(self):
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'suse', 'epochs': {}},
'oslo.config', ('>=', '1.2.3')),
'python-oslo.config >= 1.2.3'
)
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'fedora', 'epochs': {}},
'oslo.config', ('==', '1.2.3~a0')),
'python-oslo-config == 1.2.3~a0'
)
def test_context_py2pkg_pgkname_and_version_and_epoch(self):
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'suse', 'epochs': {'oslo.config': '4'}},
'oslo.config', ('>=', '1.2.3')),
'python-oslo.config >= 4:1.2.3'
)
def test_context_py2pkg_pgkname_and_epoch_no_version(self):
self.assertEqual(
renderspec._context_py2pkg(
{'spec_style': 'suse', 'epochs': {'oslo.config': '4'}},
'oslo.config'),
'python-oslo.config'
)
renderspec._context_py2pkg(context, pkg_name, pkg_version),
expected_result)
def test_context_epoch_without_epochs(self):
self.assertEqual(
renderspec._context_epoch(
{'spec_style': 'suse', 'epochs': {}}, 'oslo.config'),
0
)
{'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
'oslo.config'), 0)
def test_context_epoch_with_epochs(self):
self.assertEqual(
renderspec._context_epoch(
{'spec_style': 'suse', 'epochs': {'oslo.config': 4}},
'oslo.config'),
4
)
{'spec_style': 'suse', 'epochs': {'oslo.config': 4},
'requirements': {}}, 'oslo.config'), 4)
@ddt
class RenderspecTemplateFilterTests(unittest.TestCase):
def setUp(self):
"""create a Jinja2 environment and register the standard filters"""
self.env = Environment()
renderspec._env_register_filters_and_globals(self.env)
def test_render_filter_py2pkg_oldstyle(self):
template = self.env.from_string("{{ 'requests' | py2pkg }} >= 2.8.1")
@data(
# old style
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
"{{ 'requests' | py2pkg }}", "python-requests"),
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
"{{ 'requests' | py2pkg }} >= 2.8.1", "python-requests >= 2.8.1"),
({'spec_style': 'suse', 'epochs': {},
'requirements': {'requests': '1.2.3'}},
"{{ 'requests' | py2pkg }} >= 2.8.1", "python-requests >= 2.8.1"),
)
@unpack
def test_render_filter_py2pkg(self, context, string, expected_result):
template = self.env.from_string(string)
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'python-requests >= 2.8.1')
def test_render_filter_py2pkg(self):
template = self.env.from_string(
"{{ 'requests' | py2pkg }}")
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'python-requests')
def test_render_filter_py2pkg_with_version(self):
template = self.env.from_string(
"{{ 'requests' | py2pkg(('>=', '2.8.1')) }}")
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'python-requests >= 2.8.1')
def test_render_filter_py2pkg_with_version_and_epoch(self):
template = self.env.from_string(
"{{ 'requests' | py2pkg(('>=', '2.8.1')) }}")
self.assertEqual(
template.render(spec_style='suse', epochs={'requests': '1'}),
'python-requests >= 1:2.8.1')
def test_render_filter_epoch_without_epochs(self):
template = self.env.from_string(
"{{ 'requests' | epoch }}")
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'0')
def test_render_filter_epoch_with_epochs(self):
template = self.env.from_string(
"{{ 'requests' | epoch }}")
self.assertEqual(
template.render(spec_style='suse', epochs={'requests': '1'}),
'1')
template.render(**context),
expected_result)
@ddt
class RenderspecTemplateFunctionTests(unittest.TestCase):
def setUp(self):
"""create a Jinja2 environment and register the standard filters"""
@ -155,44 +135,88 @@ class RenderspecTemplateFunctionTests(unittest.TestCase):
template = self.env.from_string(
"{{ license('Apache-2.0') }}")
self.assertEqual(
template.render(spec_style='fedora', epochs={}),
template.render(spec_style='fedora', epochs={}, requirements={}),
'ASL 2.0')
def test_render_func_py2pkg(self):
template = self.env.from_string(
"{{ py2pkg('requests') }}")
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'python-requests')
@data(
# plain
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
"{{ py2pkg('requests') }}", "python-requests"),
# with version
({'spec_style': 'suse', 'epochs': {}, 'requirements': {}},
"{{ py2pkg('requests', ('>=', '2.8.1')) }}",
"python-requests >= 2.8.1"),
# with version, with epoch
({'spec_style': 'suse', 'epochs': {'requests': 4}, 'requirements': {}},
"{{ py2pkg('requests', ('>=', '2.8.1')) }}",
"python-requests >= 4:2.8.1"),
# with version, with epoch, with requirements
({'spec_style': 'suse', 'epochs': {'requests': 4},
'requirements': {'requests': '1.2.3'}},
"{{ py2pkg('requests', ('>=', '2.8.1')) }}",
"python-requests >= 4:2.8.1"),
# without version, with epoch, with requirements
({'spec_style': 'suse', 'epochs': {'requests': 4},
'requirements': {'requests': '1.2.3'}},
"{{ py2pkg('requests') }}",
"python-requests >= 4:1.2.3"),
def test_render_func_py2pkg_with_version(self):
template = self.env.from_string(
"{{ py2pkg('requests', ('>=', '2.8.1')) }}")
)
@unpack
def test_render_func_py2pkg(self, context, string, expected_result):
template = self.env.from_string(string)
self.assertEqual(
template.render(spec_style='suse', epochs={}),
'python-requests >= 2.8.1')
def test_render_func_py2pkg_with_version_and_epoch(self):
template = self.env.from_string(
"{{ py2pkg('requests', ('>=', '2.8.1')) }}")
self.assertEqual(
template.render(spec_style='suse', epochs={'requests': '1'}),
'python-requests >= 1:2.8.1')
template.render(**context),
expected_result)
def test_render_func_epoch_without_epochs(self):
template = self.env.from_string(
"Epoch: {{ epoch('requests') }}")
self.assertEqual(
template.render(spec_style='suse', epochs={}),
template.render(spec_style='suse', epochs={}, requirements={}),
'Epoch: 0')
def test_render_func_epoch_with_epochs(self):
template = self.env.from_string(
"Epoch: {{ epoch('requests') }}")
self.assertEqual(
template.render(spec_style='suse', epochs={'requests': 1}),
template.render(spec_style='suse', epochs={'requests': 1},
requirements={}),
'Epoch: 1')
class RenderspecVersionsTests(unittest.TestCase):
def test_without_version(self):
requires = renderspec.versions.get_requirements(
['# a comment', '', ' ', 'pyasn1 # BSD', 'Paste'])
self.assertEqual(requires, {})
def test_with_single_version(self):
requires = renderspec.versions.get_requirements(
['paramiko>=1.16.0 # LGPL'])
self.assertEqual(requires, {'paramiko': '1.16.0'})
def test_with_multiple_versions(self):
requires = renderspec.versions.get_requirements(
['sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 # BSD'])
self.assertEqual(requires, {'sphinx': '1.1.2'})
def test_with_multiple_versions_and_invalid_lowest(self):
requires = renderspec.versions.get_requirements(
['sphinx>=1.1.2,!=1.1.0,!=1.3b1,<1.3 # BSD'])
self.assertEqual(requires, {'sphinx': '1.1.2'})
def test_with_single_marker(self):
requires = renderspec.versions.get_requirements(
["pywin32>=1.0;sys_platform=='win32' # PSF"])
self.assertEqual(requires, {})
def test_with_multiple_markers(self):
requires = renderspec.versions.get_requirements(
["""pyinotify>=0.9.6;sys_platform!='win32' and \
sys_platform!='darwin' and sys_platform!='sunos5' # MIT"""])
self.assertEqual(requires, {'pyinotify': '0.9.6'})
if __name__ == '__main__':
unittest.main()