From f7fbf480daf8dbb09650a6de4f21c1a487bfdedb Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 30 Jul 2013 14:35:34 -0400 Subject: [PATCH] one global-requirements.txt to rule them all ... and in the darkness bind them. This moves the contents of requirements.txt and test-requirements.txt to a single file, global-requirements.txt. This means that regardless of whether a requirement is in either file it will get checked. This massively simplifies the checking and upgrading of req and test-req files. This also adds in a unit testing framework for the update.py script to ensure it does what we think it does. This is accomplished by making a fake tree with a set of req and test-req files, running update.py, and ensuring the updates were made that we expected. It includes testing for the oslo url case, as well as test-requires. Change-Id: Ib9b86ade4cb8317509e218aec31f32e5d08f4035 --- .testr.conf | 8 ++ requirements.txt => global-requirements.txt | 47 +++++++ test-requirements.txt | 42 ------- tests/__init__.py | 0 tests/files/gr-base.txt | 131 ++++++++++++++++++++ tests/files/project-with-oslo-tar.txt | 35 ++++++ tests/files/project.txt | 43 +++++++ tests/files/test-project.txt | 18 +++ tests/test_update.py | 83 +++++++++++++ tox.ini | 12 ++ update.py | 43 +++---- 11 files changed, 397 insertions(+), 65 deletions(-) create mode 100644 .testr.conf rename requirements.txt => global-requirements.txt (64%) create mode 100644 tests/__init__.py create mode 100644 tests/files/gr-base.txt create mode 100644 tests/files/project-with-oslo-tar.txt create mode 100644 tests/files/project.txt create mode 100644 tests/files/test-project.txt create mode 100644 tests/test_update.py create mode 100644 tox.ini diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000000..3aa8cae2ad --- /dev/null +++ b/.testr.conf @@ -0,0 +1,8 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION + +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/requirements.txt b/global-requirements.txt similarity index 64% rename from requirements.txt rename to global-requirements.txt index 1e66d266c8..ee32ef59c0 100644 --- a/requirements.txt +++ b/global-requirements.txt @@ -81,3 +81,50 @@ websockify>=0.5.1,<0.6 wsgiref>=0.1.2 WSME>=0.5b2 xattr>=0.4 +# Testing tools below, which are typically in test-requires.txt + +cliff-tablib>=1.0 +configobj +coverage>=3.6 +discover +django-nose +docutils==0.9.1 +feedparser +fixtures>=0.3.12 +flake8==2.0 +hacking>=0.5.6,<0.7 +hgtools # dependency of pytest-runner that is only in setup_requires +hp3parclient>=1.0.0 +httpretty>=0.6.3 +keyring +mock>=0.8.0 +mox>=0.5.3 +MySQL-python +nose +nose-exclude +nosehtmloutput>=0.0.3 +nosexcover +openstack.nose_plugin>=0.7 +pep8==1.4.5 +psycopg2 +pyflakes==0.7.2 +pylint==0.25.2 +pysendfile==2.0.0 +pysqlite +python-ldap==2.3.13 +python-subunit +pytest-runner # dependency of keyring that is only in setup_requires +pyzmq==2.2.0.1 +redis +selenium +sphinx>=1.1.2 +sphinxcontrib-httpdomain +sphinxcontrib-pecanwsme>=0.2 +oslo.sphinx +swift +testrepository>=0.0.17 +testresources<0.3 +testscenarios>=0.4,<0.5 +testtools>=0.9.32 +unittest2 +WebTest==1.3.3 diff --git a/test-requirements.txt b/test-requirements.txt index 11fc488d9b..7228a100f9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,45 +1,3 @@ -cliff-tablib>=1.0 -configobj -coverage>=3.6 -discover -django-nose -docutils==0.9.1 -feedparser -fixtures>=0.3.12 -flake8==2.0 -hacking>=0.5.6,<0.7 -hgtools # dependency of pytest-runner that is only in setup_requires -hp3parclient>=1.0.0 -httpretty>=0.6.3 -keyring -mock>=0.8.0 -mox>=0.5.3 -MySQL-python -nose -nose-exclude -nosehtmloutput>=0.0.3 -nosexcover -openstack.nose_plugin>=0.7 -pep8==1.4.5 -psycopg2 -pyflakes==0.7.2 -pylint==0.25.2 -pysendfile==2.0.0 -pysqlite -python-ldap==2.3.13 -python-subunit -pytest-runner # dependency of keyring that is only in setup_requires -pyzmq==2.2.0.1 -redis -selenium -sphinx>=1.1.2 -sphinxcontrib-httpdomain -sphinxcontrib-pecanwsme>=0.2 -oslo.sphinx -swift testrepository>=0.0.17 -testresources<0.3 testscenarios>=0.4,<0.5 testtools>=0.9.32 -unittest2 -WebTest==1.3.3 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/files/gr-base.txt b/tests/files/gr-base.txt new file mode 100644 index 0000000000..8a0229e439 --- /dev/null +++ b/tests/files/gr-base.txt @@ -0,0 +1,131 @@ +alembic>=0.4.1 +amqplib>=0.6.1 +anyjson>=0.3.3 +argparse +Babel>=0.9.6 +boto>=2.4.0 +cffi +Cheetah>=2.4.4 +cliff>=1.4 +d2to1>=0.2.10,<0.3 +Django>=1.4,<1.6 +django_compressor>=1.3 +django_openstack_auth>=1.1.0 +dnspython>=1.9.4 +eventlet>=0.12.0 +extras +Flask==0.9 +greenlet>=0.3.2 +happybase>=0.4 +httplib2 +iso8601>=0.1.4 +Jinja2 +jsonrpclib +jsonschema>=1.0.0,!=1.4.0,<2 +kazoo>=0.9,<=1.1 +lesscpy>=0.9h +kombu>=2.4.8 +lockfile>=0.8 +lxml>=2.3 +msgpack-python +netaddr +netifaces>=0.5 +oauth2 +oslo.config>=1.1.0 +pam>=0.1.4 +paramiko>=1.8.0 +passlib +Paste +PasteDeploy>=1.5.0 +pbr>=0.5.16,<0.6 +pecan>=0.2.0 +pip>=1.0 +PrettyTable>=0.6,<0.8 +psutil<1.0 +pyasn1 +pycrypto>=2.6 +pymongo>=2.4 +pyOpenSSL +pyparsing>=1.5.7,<2.0 +# OpenStack clients. None of these should have an upper bound +# as that has implications for testing in the gate. An exception +# is currently being made for neutron client because of the need +# for an incompatible change in their next release. +python-cinderclient>=1.0.4 +python-ceilometerclient>=1.0.2 +python-heatclient>=0.2.3 +python-glanceclient>=0.9.0 +python-keystoneclient>=0.3.0 +python-memcached +python-neutronclient>=2.2.3,<3 +python-novaclient>=2.12.0 +python-quantumclient>=2.2.0 +python-swiftclient>=1.2 +python-troveclient +pytz>=2010h +pyudev +PyYAML>=3.1.0 +qpid-python +requests>=1.1,<1.2.3 +Routes>=1.12.3 +setuptools_git>=0.4 +simplejson>=2.0.9 +six +sockjs-tornado>=1.0.0,<2.0.0 +SQLAlchemy>=0.7,<=0.7.99 +sqlalchemy-migrate>=0.7 +stevedore>=0.10 +suds>=0.4 +warlock>=0.7.0,<2 +WebOb>=1.2.3,<1.3 +websockify>=0.5.1,<0.6 +wsgiref>=0.1.2 +WSME>=0.5b2 +xattr>=0.4 +# Testing tools below, which are typically in test-requires.txt + +cliff-tablib>=1.0 +configobj +coverage>=3.6 +discover +django-nose +docutils==0.9.1 +feedparser +fixtures>=0.3.12 +flake8==2.0 +hacking>=0.5.6,<0.7 +hgtools # dependency of pytest-runner that is only in setup_requires +hp3parclient>=1.0.0 +httpretty>=0.6.3 +keyring +mock>=0.8.0 +mox>=0.5.3 +MySQL-python +nose +nose-exclude +nosehtmloutput>=0.0.3 +nosexcover +openstack.nose_plugin>=0.7 +pep8==1.4.5 +psycopg2 +pyflakes==0.7.2 +pylint==0.25.2 +pysendfile==2.0.0 +pysqlite +python-ldap==2.3.13 +python-subunit +pytest-runner # dependency of keyring that is only in setup_requires +pyzmq==2.2.0.1 +redis +selenium +sphinx>=1.1.2 +sphinxcontrib-httpdomain +sphinxcontrib-pecanwsme>=0.2 +oslo.sphinx +swift +testrepository>=0.0.17 +testresources<0.3 +testscenarios>=0.4,<0.5 +testtools>=0.9.32 +unittest2 +WebTest==1.3.3 diff --git a/tests/files/project-with-oslo-tar.txt b/tests/files/project-with-oslo-tar.txt new file mode 100644 index 0000000000..13980e7896 --- /dev/null +++ b/tests/files/project-with-oslo-tar.txt @@ -0,0 +1,35 @@ +d2to1>=0.2.10,<0.3 +pbr>=0.5.16,<0.6 +SQLAlchemy>=0.7.8,<0.7.99 +Cheetah>=2.4.4 +amqplib>=0.6.1 +anyjson>=0.2.4 +argparse +boto +eventlet>=0.9.17 +kombu>=1.0.4 +lxml>=2.3 +routes>=1.12.3 +WebOb==1.2.3 +greenlet>=0.3.1 +PasteDeploy>=1.5.0 +paste +sqlalchemy-migrate>=0.7.2 +netaddr>=0.7.6 +suds>=0.4 +paramiko +pyasn1 +Babel>=0.9.6 +iso8601>=0.1.4 +requests>=1.1,<1.2.1 # order-dependent python-cinderclient req cap, bug 1182271 +python-cinderclient>=1.0.1 +python-neutronclient>=2.2.3,<3.0.0 +python-glanceclient>=0.9.0 +python-keystoneclient>=0.2.0 +six +stevedore>=0.10 +websockify<0.4 +pyparsing>=1.5.7,<2.0 # order-dependent python-quantumclient req, bug 1191866 + +-f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3 +oslo.config>=1.2.0a3 diff --git a/tests/files/project.txt b/tests/files/project.txt new file mode 100644 index 0000000000..f8ae7931f0 --- /dev/null +++ b/tests/files/project.txt @@ -0,0 +1,43 @@ +# The greenlet package must be compiled with gcc and needs +# the Python.h headers. Make sure you install the python-dev +# package to get the right headers... +greenlet>=0.3.1 + +# < 0.8.0/0.8 does not work, see https://bugs.launchpad.net/bugs/1153983 +SQLAlchemy>=0.7.8,<=0.7.99 +anyjson>=0.3.3 +eventlet>=0.9.12 +PasteDeploy +routes +WebOb>=1.2 +wsgiref +argparse +boto +sqlalchemy-migrate>=0.7 +httplib2 +kombu>2.4.7 +pycrypto>=2.1.0alpha1 +iso8601>=0.1.4 +oslo.config>=1.1.0 + + +# For Swift storage backend. +python-swiftclient>=1.2,<2 + +# Note you will need gcc buildtools installed and must +# have installed libxml headers for lxml to be successfully +# installed using pip, therefore you will need to install the +# libxml2-dev and libxslt-dev Ubuntu packages. +lxml + +# For paste.util.template used in keystone.common.template +Paste + +passlib +jsonschema +python-cinderclient>=1.0.4 +python-keystoneclient>=0.2.0 +pyOpenSSL + +# Required by openstack.common libraries +six diff --git a/tests/files/test-project.txt b/tests/files/test-project.txt new file mode 100644 index 0000000000..15525ccc92 --- /dev/null +++ b/tests/files/test-project.txt @@ -0,0 +1,18 @@ +hacking>=0.5.6,<0.7 +coverage>=3.6 +discover +feedparser +fixtures>=0.3.12 +mox==0.5.3 +MySQL-python +psycopg2 +pylint==0.25.2 +# Imported by ldapdns so required to generate +# the sample configuration file +python-ldap==2.3.13 +python-subunit +setuptools_git>=0.4 +sphinx>=1.1.2 +oslo.sphinx +testrepository>=0.0.13 +testtools>=0.9.27 diff --git a/tests/test_update.py b/tests/test_update.py new file mode 100644 index 0000000000..14bce5e6e2 --- /dev/null +++ b/tests/test_update.py @@ -0,0 +1,83 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import os.path +import shutil +import subprocess +import sys +import tempfile +import testtools + + +def _load_req(fname): + with open(fname) as f: + content = map(lambda x: x.rstrip(), f.readlines()) + print content + return content + + +class UpdateTest(testtools.TestCase): + + def setUp(self): + super(UpdateTest, self).setUp() + self.dir = tempfile.mkdtemp() + self.project_dir = os.path.join(self.dir, "project") + self.oslo_dir = os.path.join(self.dir, "project_with_oslo") + + self.req_file = os.path.join(self.dir, "global-requirements.txt") + self.proj_file = os.path.join(self.project_dir, "requirements.txt") + self.oslo_file = os.path.join(self.oslo_dir, "requirements.txt") + self.proj_test_file = os.path.join(self.project_dir, + "test-requirements.txt") + os.mkdir(self.project_dir) + os.mkdir(self.oslo_dir) + + shutil.copy("tests/files/gr-base.txt", self.req_file) + shutil.copy("tests/files/project-with-oslo-tar.txt", self.oslo_file) + shutil.copy("tests/files/project.txt", self.proj_file) + shutil.copy("tests/files/test-project.txt", self.proj_test_file) + shutil.copy("update.py", os.path.join(self.dir, "update.py")) + + # now go call update and see what happens + os.chdir(self.dir) + subprocess.call([sys.executable, "update.py", "project"]) + subprocess.call([sys.executable, "update.py", "project_with_oslo"]) + + def test_requirements(self): + reqs = _load_req(self.req_file) + self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs) + + def test_project(self): + reqs = _load_req(self.proj_file) + # ensure various updates take + self.assertIn("jsonschema>=1.0.0,!=1.4.0,<2", reqs) + self.assertIn("python-keystoneclient>=0.3.0", reqs) + self.assertIn("SQLAlchemy>=0.7,<=0.7.99", reqs) + + def test_project_with_oslo(self): + reqs = _load_req(self.oslo_file) + oslo_tar = ("-f http://tarballs.openstack.org/oslo.config/" + "oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3") + self.assertIn(oslo_tar, reqs) + self.assertIn("oslo.config>=1.1.0", reqs) + + def test_test_project(self): + reqs = _load_req(self.proj_test_file) + self.assertIn("testtools>=0.9.32", reqs) + self.assertIn("testrepository>=0.0.17", reqs) + # make sure we didn't add something we shouldn't + self.assertNotIn("sphinxcontrib-pecanwsme>=0.2", reqs) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..4d833f9e18 --- /dev/null +++ b/tox.ini @@ -0,0 +1,12 @@ +[tox] +envlist = py27 + +[testenv] +setenv = VIRTUAL_ENV={envdir} + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C +deps = -U + -r{toxinidir}/test-requirements.txt +commands = + python setup.py testr --slowest --testr-args='{posargs}' diff --git a/update.py b/update.py index 5db35bc85a..ba09a99287 100644 --- a/update.py +++ b/update.py @@ -66,30 +66,12 @@ def _parse_reqs(filename): return reqs -def _copy_requires(source_path, dest_dir): - """Copy requirements files.""" - - target_map = { - 'requirements.txt': ('requirements.txt', 'tools/pip-requires'), - 'test-requirements.txt': ( - 'test-requirements.txt', 'tools/test-requires'), - } - for dest in target_map[source_path]: - dest_path = os.path.join(dest_dir, dest) - if os.path.exists(dest_path): - break - - # Catch the fall through - if not os.path.exists(dest_path): - # This can happen, we try all paths - return - - source_reqs = _parse_reqs(source_path) - +def _sync_requirements_file(source_reqs, dest_path): + dest_reqs = [] with open(dest_path, 'r') as dest_reqs_file: dest_reqs = dest_reqs_file.readlines() - print "Syncing %s" % source_path + print "Syncing %s" % dest_path with open(dest_path, 'w') as new_reqs: for old_line in dest_reqs: @@ -111,9 +93,24 @@ def _copy_requires(source_path, dest_dir): new_reqs.write("%s\n" % source_reqs[old_pip]) +def _copy_requires(source_path, dest_dir): + """Copy requirements files.""" + + source_reqs = _parse_reqs(source_path) + + target_files = ( + 'requirements.txt', 'tools/pip-requires', + 'test-requirements.txt', 'tools/test-requires') + + for dest in target_files: + dest_path = os.path.join(dest_dir, dest) + if os.path.exists(dest_path): + print "_sync_requirements_file(%s, %s)" % (source_reqs, dest_path) + _sync_requirements_file(source_reqs, dest_path) + + def main(argv): - for req in ('requirements.txt', 'test-requirements.txt'): - _copy_requires(req, argv[0]) + _copy_requires('global-requirements.txt', argv[0]) if __name__ == "__main__":