From e545f18303ce00d2bb302eacc7ce8de388b45c17 Mon Sep 17 00:00:00 2001 From: "Ivan A. Melnikov" Date: Mon, 12 Aug 2013 17:30:42 +0300 Subject: [PATCH] Package tests We put component unit tests to subpackage to make it possible to install and run them on another system. Implements: blueprint package-unit-tests Change-Id: I77cbcd3545566ac295d873ec8b057945ad88e5f8 --- anvil/components/base_install.py | 10 +- anvil/packaging/base.py | 8 +- anvil/packaging/helpers/pip_helper.py | 38 ++++---- anvil/packaging/helpers/py2rpm_helper.py | 5 +- anvil/packaging/yum.py | 43 ++++++--- conf/distros/rhel.yaml | 4 + .../packaging/specs/install_tests.sh | 52 ++++++++++ .../packaging/specs/openstack-cinder.spec | 39 +++++++- .../packaging/specs/openstack-glance.spec | 50 ++++++++-- .../packaging/specs/openstack-keystone.spec | 41 +++++++- .../packaging/specs/openstack-neutron.spec | 52 +++++++++- .../packaging/specs/openstack-nova.spec | 50 +++++++++- .../packaging/specs/openstack-trove.spec | 35 +++++++ .../packaging/specs/python-commonclient.spec | 41 +++++++- tools/py2rpm | 95 ++++++++++++++++++- 15 files changed, 505 insertions(+), 58 deletions(-) create mode 100644 conf/templates/packaging/specs/install_tests.sh diff --git a/anvil/components/base_install.py b/anvil/components/base_install.py index b5bf809f..b037200d 100644 --- a/anvil/components/base_install.py +++ b/anvil/components/base_install.py @@ -135,6 +135,10 @@ class PythonInstallComponent(PkgInstallComponent): sh.joinpths(tools_dir, 'pip-requires'), sh.joinpths(app_dir, 'requirements.txt'), ] + self.test_requires_files = [ + sh.joinpths(tools_dir, 'test-requires'), + sh.joinpths(app_dir, 'test-requirements.txt'), + ] if self.get_bool_option('use_tests_requires', default_value=True): self.requires_files.append(sh.joinpths(tools_dir, 'test-requires')) self.requires_files.append(sh.joinpths(app_dir, @@ -166,7 +170,11 @@ class PythonInstallComponent(PkgInstallComponent): @property def egg_info(self): - return pip_helper.get_directory_details(self.get_option('app_dir')) + egg_info = pip_helper.get_directory_details(self.get_option('app_dir')).copy() + read_reqs = pip_helper.read_requirement_files + egg_info['dependencies'] = read_reqs(self.requires_files) + egg_info['test_dependencies'] = read_reqs(self.test_requires_files) + return egg_info class PkgUninstallComponent(base.Component): diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index caa18b0d..30582740 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -137,10 +137,10 @@ class DependencyHandler(object): requires_files = [] extra_pips = [] for i in self.instances: - try: - requires_files.extend(i.requires_files) - except AttributeError: - pass + requires_files.extend(getattr(i, 'requires_files', ())) + if i.get_bool_option('use_tests_requires', default_value=True): + requires_files.extend(getattr(i, 'test_requires_files', ())) + # Ensure we include any extra pips that are desired. i_extra_pips = i.get_option("pips") or [] for i_pip in i_extra_pips: diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 92b04b91..c6912608 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -124,31 +124,37 @@ def get_archive_details(filename): return details +SKIP_LINES = ('#', '-e', '-f', 'http://', 'https://') + + def _skip_requirement(line): - # Skip blank lines or comment lines - if not len(line): - return True - if line.startswith("#"): - return True - # Skip editables also... - if line.lower().startswith('-e'): - return True - # Skip http types also... - if line.lower().startswith('http://'): - return True - return False + return not len(line) or any(line.startswith(a) for a in SKIP_LINES) + + +OPESTACK_TARBALLS_RE = re.compile(r'http://tarballs.openstack.org/([^/]+)/') def parse_requirements(contents, adjust=False): lines = [] for line in contents.splitlines(): line = line.strip() + if 'http://' in line: + m = OPESTACK_TARBALLS_RE.search(line) + if m: + line = m.group(1) if not _skip_requirement(line): lines.append(line) - requires = [] - for req in pkg_resources.parse_requirements(lines): - requires.append(req) - return requires + return pkg_resources.parse_requirements(lines) + + +def read_requirement_files(files): + result = [] + for filename in files: + if sh.isfile(filename): + LOG.debug('Parsing requirements from %s', filename) + with open(filename) as f: + result.extend(parse_requirements(f.read())) + return result class Helper(object): diff --git a/anvil/packaging/helpers/py2rpm_helper.py b/anvil/packaging/helpers/py2rpm_helper.py index 5c6bf6a5..18b4566b 100644 --- a/anvil/packaging/helpers/py2rpm_helper.py +++ b/anvil/packaging/helpers/py2rpm_helper.py @@ -107,10 +107,13 @@ class Helper(object): logger=LOG) self._execute_make(makefile_path, marks_dir, jobs) - def build_srpm(self, source, log_filename, release=None): + def build_srpm(self, source, log_filename, + release=None, with_tests=False): cmdline = self._start_cmdline() + ["--source-only"] if release is not None: cmdline.extend(["--release", release]) + if with_tests: + cmdline.append("--with-tests") cmdline.extend(["--", source]) out_filename = sh.joinpths(self._log_dir, "py2rpm-build-%s.log" % log_filename) diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index e0b14a2a..fd8eedd0 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -129,6 +129,12 @@ class YumDependencyHandler(base.DependencyHandler): if version_suffix and not version_suffix.startswith('.'): version_suffix = '.' + version_suffix params['version_suffix'] = version_suffix + tests_package = instance.get_option('tests_package', default_value={}) + + params["no_tests"] = 0 if tests_package.get('enabled', True) else 1 + params["exclude_from_test_env"] = ['./bin', './build*'] + params["exclude_from_test_env"].extend( + tests_package.get("exclude_from_env", ())) return params def _create_rpmbuild_subdirs(self): @@ -362,19 +368,24 @@ class YumDependencyHandler(base.DependencyHandler): jobs=self._jobs) def _write_spec_file(self, instance, rpm_name, template_name, params): - requires_what = params.get('requires') - if not requires_what: - requires_what = [] - requires_python = [] - try: - requires_python.extend(instance.egg_info['dependencies']) - except AttributeError: - pass - if requires_python: - requires_what.extend( - self.py2rpm_helper.convert_names_to_rpm(requires_python, False)) - params['requires'] = requires_what + requires_what = params.get('requires', []) + test_requires_what = params.get('test_requires', []) + egg_info = getattr(instance, 'egg_info', None) + if egg_info: + def ei_names(key): + requires_python = [str(req) for req in egg_info[key]] + return self.py2rpm_helper.convert_names_to_rpm(requires_python, False) + + requires_what.extend(ei_names('dependencies')) + test_requires_what.extend(ei_names('test_dependencies')) + params["requires"] = requires_what + params["test_requires"] = test_requires_what params["epoch"] = self.OPENSTACK_EPOCH + params["part_fn"] = lambda filename: sh.joinpths( + settings.TEMPLATE_DIR, + self.SPEC_TEMPLATE_DIR, + filename) + content = utils.load_template(self.SPEC_TEMPLATE_DIR, template_name)[1] spec_filename = sh.joinpths(self.rpmbuild_dir, "SPECS", "%s.spec" % rpm_name) sh.write_file(spec_filename, utils.expand_template(content, params), @@ -552,9 +563,11 @@ class YumDependencyHandler(base.DependencyHandler): template_name, params) self._build_from_spec(instance, spec_filename, patches) else: - self.py2rpm_helper.build_srpm(source=instance.get_option("app_dir"), - log_filename=instance.name, - release=params.get("release")) + self.py2rpm_helper.build_srpm( + source=instance.get_option("app_dir"), + log_filename=instance.name, + release=params.get("release"), + with_tests=not params.get("no_tests")) def _get_rpm_names(self, from_deps=True, from_instances=True): desired_rpms = [] diff --git a/conf/distros/rhel.yaml b/conf/distros/rhel.yaml index bc2c9164..1639a3fd 100644 --- a/conf/distros/rhel.yaml +++ b/conf/distros/rhel.yaml @@ -246,6 +246,10 @@ components: # installed in rhel uses a old version of crypto which # other components actually can't use. This sucks... - name: paramiko + test_requires: + # NOTE(imelnikov): nova testcases require importlib, which was not part + # of python standard library as of python 2.6. + - importlib daemon_to_package: api: openstack-nova-api conductor: openstack-nova-conductor diff --git a/conf/templates/packaging/specs/install_tests.sh b/conf/templates/packaging/specs/install_tests.sh new file mode 100644 index 00000000..e60a8a34 --- /dev/null +++ b/conf/templates/packaging/specs/install_tests.sh @@ -0,0 +1,52 @@ +#* + Cheetah template to be included to %install spec section +*# +# Package test environment +install -d -m 755 %{buildroot}%{tests_data_dir} +tar -cf "%{buildroot}%{tests_data_dir}/test_env.tar" \ + --exclude "./%{python_name}" \ +#for i in $exclude_from_test_env + --exclude "${i}" \ +#end for + --exclude-vcs . +find "./%{python_name}" -type d -name tests | while read testdir; do + tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \ +#for i in $exclude_from_test_env + --exclude "${i}" \ +#end for + "\$testdir" +done +if [ -r "./%{python_name}/test.py" ]; then + tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \ + ./%{python_name}/test.py +fi +gzip -9 "%{buildroot}%{tests_data_dir}/test_env.tar" + + +# Script that prepares test environment: +#raw +cat > %{buildroot}%{_bindir}/%{python_name}-make-test-env <<"EOF" +#!/bin/bash + +set -e + +if [ -z "$1" ] || [ "$1" == "--help" ] ; then + echo "Usage: $0 [dir]" + echo " $0 --tmpdir" +fi + +if [ "$1" == "--tmpdir" ]; then + target_dir=$(mktemp -dt "${0##*/}.XXXXXXXX") + echo "Created temporary directory: $target_dir" +else + target_dir="$1" +fi + +cd "$target_dir" +tar -xzf "%{tests_data_dir}/test_env.tar.gz" +cp -a %{python_sitelib}/%{python_name} . +ln -s /usr/bin ./bin + +EOF +chmod 0755 %{buildroot}%{_bindir}/%{python_name}-make-test-env +#end raw diff --git a/conf/templates/packaging/specs/openstack-cinder.spec b/conf/templates/packaging/specs/openstack-cinder.spec index d590eaac..79388119 100644 --- a/conf/templates/packaging/specs/openstack-cinder.spec +++ b/conf/templates/packaging/specs/openstack-cinder.spec @@ -7,6 +7,8 @@ %global python_name cinder %global daemon_prefix openstack-cinder %global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 6) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -74,6 +76,29 @@ access block storage volumes for use by Virtual Machine instances. This package contains the cinder Python library. + +%if ! 0%{?no_tests} +%package -n python-%{python_name}-tests +Summary: Tests for Cinder +Group: Development/Libraries + +Requires: %{name} = %{epoch}:%{version}-%{release} +Requires: python-%{python_name} = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-%{python_name}-tests +OpenStack Volume (codename Cinder) provides services to manage and +access block storage volumes for use by Virtual Machine instances. + +This package contains unit and functional tests for Cinder, with +simple runner (%{python_name}-make-test-env). +%endif + + %if 0%{?with_doc} %package doc Summary: Documentation for OpenStack Volume @@ -125,6 +150,12 @@ sed -i '/setup_requires/d; /install_requires/d; /dependency_links/d' setup.py %install %{__python} setup.py install -O1 --skip-build --root %{buildroot} +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif + # docs generation requires everything to be installed first export PYTHONPATH="$PWD:$PYTHONPATH" @@ -182,8 +213,9 @@ install -p -D -m 644 etc/cinder/rootwrap.d/* %{buildroot}%{_datarootdir}/cinder/ # Remove unneeded in production stuff rm -f %{buildroot}%{_bindir}/cinder-debug +rm -f %{buildroot}%{python_sitelib}/cinder/test.py* rm -fr %{buildroot}%{python_sitelib}/cinder/tests/ -rm -fr %{buildroot}%{python_sitelib}/run_tests.* +rm -f %{buildroot}%{python_sitelib}/run_tests.* rm -f %{buildroot}/usr/share/doc/cinder/README* @@ -250,6 +282,11 @@ fi %{python_sitelib}/cinder %{python_sitelib}/cinder-%{os_version}*.egg-info +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif %if 0%{?with_doc} %files doc diff --git a/conf/templates/packaging/specs/openstack-glance.spec b/conf/templates/packaging/specs/openstack-glance.spec index 5bdbb9a8..0b34265c 100644 --- a/conf/templates/packaging/specs/openstack-glance.spec +++ b/conf/templates/packaging/specs/openstack-glance.spec @@ -6,6 +6,8 @@ %global python_name glance %global daemon_prefix openstack-glance %global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 6) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -66,13 +68,34 @@ Group: Applications/System Requires: ${i} #end for -#raw %description -n python-glance OpenStack Image Service (code-named Glance) provides discovery, registration, and delivery services for virtual disk images. This package contains the glance Python library. + +%if ! 0%{?no_tests} +%package -n python-glance-tests +Summary: Tests for Glance +Group: Development/Libraries + +Requires: openstack-glance = %{epoch}:%{version}-%{release} +Requires: python-glance = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-glance-tests +OpenStack Image Service (code-named Glance) provides discovery, registration, +and delivery services for virtual disk images. + +This package contains the Glance unit and functional tests, with simple +runner (glance-make-test-env). +%endif + %if 0%{?with_doc} %package doc Summary: Documentation for OpenStack Glance @@ -96,13 +119,19 @@ This package contains documentation files for glance. %prep %setup -q -n %{python_name}-%{os_version} -#end raw #for $idx, $fn in enumerate($patches) %patch$idx -p1 #end for #raw +# make tests run real installed binaries +find glance/tests -name '*.py' | while read filename; do + sed -i \ + -e "s,\./bin/glance,%{_bindir}/glance,g" \ + -e "s,\('\|\"\)bin/glance,\1%{_bindir}/glance,g" \ + "$filename" +done %build %{__python} setup.py build @@ -111,9 +140,6 @@ This package contains documentation files for glance. rm -rf %{buildroot} %{__python} setup.py install -O1 --skip-build --root %{buildroot} -# Delete tests -rm -fr %{buildroot}%{python_sitelib}/tests - %if 0%{?with_doc} export PYTHONPATH="$PWD:$PYTHONPATH" pushd doc @@ -149,6 +175,11 @@ install -d -m 755 %{buildroot}%{_localstatedir}/run/glance install -d -m 755 %{buildroot}%{_localstatedir}/log/glance %endif +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif %clean rm -rf %{buildroot} @@ -188,6 +219,7 @@ fi %defattr(-,root,root,-) %doc README* LICENSE* HACKING* ChangeLog AUTHORS %{_bindir}/* +%exclude %{_bindir}/glance-make-test-env %if ! 0%{?usr_only} %{_initrddir}/* @@ -200,10 +232,15 @@ fi %dir %attr(0755, glance, nobody) %{_localstatedir}/run/glance %endif - %files -n python-glance %{python_sitelib}/* +%exclude %{python_sitelib}/glance/tests +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif %if 0%{?with_doc} %files doc @@ -211,6 +248,5 @@ fi %doc doc/build/html %endif - %changelog #end raw diff --git a/conf/templates/packaging/specs/openstack-keystone.spec b/conf/templates/packaging/specs/openstack-keystone.spec index e6c36183..9f93902f 100644 --- a/conf/templates/packaging/specs/openstack-keystone.spec +++ b/conf/templates/packaging/specs/openstack-keystone.spec @@ -6,6 +6,8 @@ %global python_name keystone %global daemon_prefix openstack-keystone %global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -49,6 +51,31 @@ Keystone is a Python implementation of the OpenStack This package contains the Keystone daemon. +%if ! 0%{?no_tests} +%package -n python-%{python_name}-tests +Summary: Tests for Keystone +Group: Development/Libraries + +Requires: %{name} = %{epoch}:%{version}-%{release} +Requires: python-%{python_name} = %{epoch}:%{version}-%{release} +# To test against modern client libraries +Requires: git +Requires: python-pbr + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-%{python_name}-tests +Keystone is a Python implementation of the OpenStack +(http://www.openstack.org) identity service API. + +This package contains unit and functional tests for Keystone, with +simple runner (%{python_name}-make-test-env). +%endif + + %if 0%{?with_doc} %package doc @@ -106,6 +133,7 @@ rm -fr doc/build/html/.doctrees doc/build/html/.buildinfo python setup.py install --prefix=%{_prefix} --root=%{buildroot} + %if ! 0%{?usr_only} install -d -m 755 %{buildroot}%{_sysconfdir}/keystone install -m 644 etc/* %{buildroot}%{_sysconfdir}/keystone @@ -119,6 +147,12 @@ install -p -D -m 755 %{SOURCE1} %{buildroot}%{_initrddir}/%{daemon_prefix} %__rm -rf %{buildroot}%{py_sitelib}/{doc,tools} +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif + %clean %__rm -rf %{buildroot} @@ -162,6 +196,12 @@ fi %{_initrddir}/* %endif +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif + %if 0%{?with_doc} %files doc %defattr(-,root,root,-) @@ -173,6 +213,5 @@ fi %doc LICENSE %{python_sitelib}/* - %changelog #endraw diff --git a/conf/templates/packaging/specs/openstack-neutron.spec b/conf/templates/packaging/specs/openstack-neutron.spec index 58be701e..a8f1bf44 100644 --- a/conf/templates/packaging/specs/openstack-neutron.spec +++ b/conf/templates/packaging/specs/openstack-neutron.spec @@ -11,6 +11,8 @@ %global python_name neutron %global daemon_prefix openstack-neutron %global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 6) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -364,6 +366,40 @@ networks. This package contains the neutron plugin that implements virtual networks using multiple other neutron plugins. + +%if ! 0%{?no_tests} +%package -n python-%{python_name}-tests +Summary: Tests for Quantum +Group: Development/Libraries + +Requires: %{name} = %{epoch}:%{version}-%{release} +Requires: %{name}-bigswitch = %{epoch}:%{version}-%{release} +Requires: %{name}-brocade = %{epoch}:%{version}-%{release} +Requires: %{name}-cisco = %{epoch}:%{version}-%{release} +Requires: %{name}-hyperv = %{epoch}:%{version}-%{release} +Requires: %{name}-linuxbridge = %{epoch}:%{version}-%{release} +Requires: %{name}-midonet = %{epoch}:%{version}-%{release} +Requires: %{name}-nicira = %{epoch}:%{version}-%{release} +Requires: %{name}-openvswitch = %{epoch}:%{version}-%{release} +Requires: %{name}-plumgrid = %{epoch}:%{version}-%{release} +Requires: %{name}-ryu = %{epoch}:%{version}-%{release} +Requires: %{name}-nec = %{epoch}:%{version}-%{release} +Requires: %{name}-metaplugin = %{epoch}:%{version}-%{release} +Requires: python-%{python_name} = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-%{python_name}-tests +Quantum provides an API to dynamically request and configure virtual +networks. + +This package contains unit and functional tests for Quantum, with +simple runner (%{python_name}-make-test-env). +%endif + %prep %setup -q -n neutron-%{os_version} #for $idx, $fn in enumerate($patches) @@ -394,12 +430,16 @@ sed -i '/setup_requires/d; /install_requires/d; /dependency_links/d' setup.py rm -rf %{buildroot} %{__python} setup.py install -O1 --skip-build --root %{buildroot} +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif + # Remove unused files rm -rf %{buildroot}%{python_sitelib}/bin rm -rf %{buildroot}%{python_sitelib}/doc rm -rf %{buildroot}%{python_sitelib}/tools -rm -rf %{buildroot}%{python_sitelib}/neutron/tests -rm -rf %{buildroot}%{python_sitelib}/neutron/plugins/*/tests rm -f %{buildroot}%{python_sitelib}/neutron/plugins/*/run_tests.* rm %{buildroot}/usr/etc/init.d/neutron-server @@ -553,6 +593,7 @@ fi %doc LICENSE %{python_sitelib}/neutron %{python_sitelib}/quantum +%exclude %{python_sitelib}/neutron/tests %exclude %{python_sitelib}/neutron/plugins/bigswitch %exclude %{python_sitelib}/neutron/plugins/brocade %exclude %{python_sitelib}/neutron/plugins/cisco @@ -574,6 +615,7 @@ fi %doc LICENSE %doc neutron/plugins/bigswitch/README %{python_sitelib}/neutron/plugins/bigswitch +%exclude %{python_sitelib}/neutron/plugins/bigswitch/tests %if ! 0%{?usr_only} %dir %{_sysconfdir}/neutron/plugins/bigswitch @@ -585,6 +627,7 @@ fi %doc LICENSE %doc neutron/plugins/brocade/README.md %{python_sitelib}/neutron/plugins/brocade +%exclude %{python_sitelib}/neutron/plugins/brocade/tests %if ! 0%{?usr_only} %dir %{_sysconfdir}/neutron/plugins/brocade @@ -740,6 +783,11 @@ fi %config(noreplace) %attr(0640, root, neutron) %{_sysconfdir}/neutron/plugins/metaplugin/*.ini %endif +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif %changelog #end raw diff --git a/conf/templates/packaging/specs/openstack-nova.spec b/conf/templates/packaging/specs/openstack-nova.spec index 98878b7a..347b8d70 100644 --- a/conf/templates/packaging/specs/openstack-nova.spec +++ b/conf/templates/packaging/specs/openstack-nova.spec @@ -8,6 +8,8 @@ %global python_name nova %global daemon_prefix openstack-nova %global os_version ${version} +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 6) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -311,8 +313,41 @@ redundant and scalable cloud computing platform. This package contains the %{name} Python library. -%if 0%{?with_doc} +%if ! 0%{?no_tests} +%package -n python-%{python_name}-tests +Summary: Tests for Nova +Group: Development/Libraries + +# Bring in all subpackages: +Requires: %{name} = %{epoch}:%{version}-%{release} +Requires: %{name}-common = %{epoch}:%{version}-%{release} +Requires: %{name}-compute = %{epoch}:%{version}-%{release} +Requires: %{name}-network = %{epoch}:%{version}-%{release} +Requires: %{name}-scheduler = %{epoch}:%{version}-%{release} +Requires: %{name}-cert = %{epoch}:%{version}-%{release} +Requires: %{name}-api = %{epoch}:%{version}-%{release} +Requires: %{name}-conductor = %{epoch}:%{version}-%{release} +Requires: %{name}-objectstore = %{epoch}:%{version}-%{release} +Requires: %{name}-console = %{epoch}:%{version}-%{release} +Requires: %{name}-cells = %{epoch}:%{version}-%{release} +Requires: python-%{python_name} = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-%{python_name}-tests +Nova is a cloud computing fabric controller (the main part +of an IaaS system). + +This package contains unit and functional tests for Nova, with +simple runner (%{python_name}-make-test-env). +%endif + + +%if 0%{?with_doc} %package doc Summary: Documentation for %{name} Group: Documentation @@ -430,7 +465,6 @@ install -p -D -m 644 %{SOURCE50} %{buildroot}%{_datarootdir}/nova/interfaces.tem # Remove unneeded in production stuff rm -f %{buildroot}%{_bindir}/nova-debug -rm -fr %{buildroot}%{python_sitelib}/nova/tests/ rm -fr %{buildroot}%{python_sitelib}/run_tests.* rm -f %{buildroot}%{_bindir}/nova-combined rm -f %{buildroot}/usr/share/doc/nova/README* @@ -438,6 +472,11 @@ rm -f %{buildroot}/usr/share/doc/nova/README* # We currently use the equivalent file from the novnc package rm -f %{buildroot}%{_bindir}/nova-novncproxy +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif %clean rm -rf %{buildroot} @@ -642,8 +681,15 @@ fi %defattr(-,root,root,-) %doc LICENSE %{python_sitelib}/nova +%exclude %{python_sitelib}/%{python_name}/tests %{python_sitelib}/nova-%{os_version}-*.egg-info +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif + %if 0%{?with_doc} %files doc %doc LICENSE doc/build/html diff --git a/conf/templates/packaging/specs/openstack-trove.spec b/conf/templates/packaging/specs/openstack-trove.spec index feb99c9e..273bb623 100644 --- a/conf/templates/packaging/specs/openstack-trove.spec +++ b/conf/templates/packaging/specs/openstack-trove.spec @@ -6,6 +6,8 @@ %global python_name trove %global daemon_prefix openstack-trove %global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests %if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} @@ -86,6 +88,28 @@ of handling complex administrative tasks. This package contains the Trove Python library. + +%if ! 0%{?no_tests} +%package -n python-%{python_name}-tests +Summary: Tests for Trove +Group: Development/Libraries + +Requires: %{name} = %{epoch}:%{version}-%{release} +Requires: python-%{python_name} = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description -n python-%{python_name}-tests +Trove is Database as a Service for OpenStack. + +This package contains unit and functional tests for Trove, with +simple runner (%{python_name}-make-test-env). +%endif + + %prep %setup -q -n %{python_name}-%{os_version} #for $idx, $fn in enumerate($patches) @@ -133,6 +157,12 @@ install -p -D -m 755 %{SOURCE1} %{buildroot}%{_initrddir}/%{daemon_prefix}-serve %__rm -rf %{buildroot}%{py_sitelib}/{doc,tools} +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif + %clean %__rm -rf %{buildroot} @@ -190,6 +220,11 @@ fi %doc LICENSE %{python_sitelib}/* +%if ! 0%{?no_tests} +%files -n python-%{python_name}-tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif %changelog #endraw diff --git a/conf/templates/packaging/specs/python-commonclient.spec b/conf/templates/packaging/specs/python-commonclient.spec index 75bd07d0..b31a5293 100644 --- a/conf/templates/packaging/specs/python-commonclient.spec +++ b/conf/templates/packaging/specs/python-commonclient.spec @@ -8,13 +8,17 @@ apiname - Identity, Compute, etc. (first uppercase) requires - list of requirements for python-* package *# + +%global python_name ${clientname}client +%global os_version $version +%global no_tests $no_tests +%global tests_data_dir %{_datarootdir}/%{python_name}-tests/ + %if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} %endif -%global os_version $version - -Name: python-${clientname}client +Name: python-%{python_name} Summary: OpenStack ${clientname.title()} Client Version: %{os_version}$version_suffix Release: $release%{?dist} @@ -45,6 +49,7 @@ BuildRequires: python-sphinx BuildRequires: make %endif +# Python requirements: #for $i in $requires Requires: ${i} #end for @@ -54,6 +59,25 @@ This is a client for the OpenStack $apiname API. There's a Python API (the ${clientname}client module), and a command-line script (${clientname}). Each implements 100% of the OpenStack $apiname API. + +%if ! 0%{?no_tests} +%package tests +Summary: Tests for OpenStack ${clientname.title()} Client +Group: Development/Libraries + +Requires: %{name} = %{epoch}:%{version}-%{release} + +# Test requirements: +#for $i in $test_requires +Requires: ${i} +#end for + +%description tests +This package contains unit and functional tests for OpenStack +${clientname.title()} Client, with runner. +%endif + + %if 0%{?enable_doc} %package doc Summary: Documentation for %{name} @@ -92,6 +116,11 @@ rm -rf %{buildroot}/%{_usr}/*client make -C docs html PYTHONPATH=%{buildroot}%{python_sitelib} %endif +%if ! 0%{?no_tests} +#end raw +#include $part_fn("install_tests.sh") +#raw +%endif %clean rm -rf %{buildroot} @@ -102,7 +131,13 @@ rm -rf %{buildroot} %doc README* LICENSE* HACKING* ChangeLog AUTHORS %{python_sitelib}/* %{_bindir}/* +%exclude %{_bindir}/%{python_name}-make-test-env +%if ! 0%{?no_tests} +%files tests +%{tests_data_dir} +%{_bindir}/%{python_name}-make-test-env +%endif %if 0%{?enable_doc} %files doc diff --git a/tools/py2rpm b/tools/py2rpm index 7981f26f..3ac09cf7 100755 --- a/tools/py2rpm +++ b/tools/py2rpm @@ -52,12 +52,59 @@ abspath_installed_files=$(readlink -f INSTALLED_FILES) done fi ) >> GATHERED_FILES -{ sed '/^DELETE_ME/d' INSTALLED_FILES; cat GATHERED_FILES; } | sort -u > INSTALLED_FILES.tmp -mv -f INSTALLED_FILES{.tmp,}""", +sed '/^DELETE_ME/d' INSTALLED_FILES >> GATHERED_FILES +sort -u GATHERED_FILES > INSTALLED_FILES +""", "clean": """rm -rf $RPM_BUILD_ROOT""", } +DEFAULT_TESTS_SCRIPTS = { + "install": """ +install -d -m 755 %{buildroot}%{tests_data_dir} +tar -cf "%{buildroot}%{tests_data_dir}/test_env.tar" \ + --exclude-vcs --exclude ./%{pkg_path} \ + --exclude './build*' --exclude './bin' --exclude './smoketest*' \ + . +if [ -d "./%{pkg_path}/tests" ]; then + tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \ + ./%{pkg_path}/tests +fi +if [ -r "./%{pkg_path}/test.py" ]; then + tar -rf "%{buildroot}%{tests_data_dir}/test_env.tar" \ + ./%{pkg_path}/test.py +fi +gzip -9 "%{buildroot}%{tests_data_dir}/test_env.tar" + + +# Make simple test runner +install -d -m 755 %{buildroot}%{_bindir} +cat > %{buildroot}%{_bindir}/%{pkg_name}-make-test-env <<"EOF" +#!/bin/bash + +set -e + +if [ -z "$1" ] || [ "$1" == "--help" ] ; then + echo "Usage: $0 [dir]" + echo " $0 --tmpdir" +fi + +if [ "$1" == "--tmpdir" ]; then + target_dir=$(mktemp -dt "${0##*/}.XXXXXXXX") + echo "Created temporary directory: $target_dir" +else + target_dir="$1" +fi + +cd "$target_dir" +tar -xzf "%{tests_data_dir}/test_env.tar.gz" +cp -a %{python_sitelib}/%{pkg_path} `dirname %{pkg_path}` +ln -s /usr/bin ./bin + +EOF +chmod 0755 %{buildroot}%{_bindir}/%{pkg_name}-make-test-env +""" +} class InstallationError(Exception): pass @@ -104,9 +151,9 @@ def egg_info_lines(source_dir, filename): return f.readlines() -def egg_info_requirements(source_dir, extras=()): +def egg_info_requirements(source_dir, extras=(), filename='requires.txt'): in_extra = None - for line in egg_info_lines(source_dir, 'requires.txt'): + for line in egg_info_lines(source_dir, filename): match = requirements_section_re.match(line.lower()) if match: in_extra = match.group(1) @@ -229,6 +276,11 @@ def create_parser(): nargs="+", default=[], help="Correspondence between Python and RPM package names") + parser.add_argument( + "--with-tests", + action="store_true", + default=False, + help="Add subpackage with tests") return parser @@ -302,6 +354,9 @@ def replacement_run(self): if writer: writer(self, ep.name, os.path.join(self.egg_info,ep.name)) self.find_sources() + if self.distribution.tests_require: + with open(os.path.join(self.egg_info, 'test-requires.txt'), 'w') as f: + f.write('\\n'.join(self.distribution.tests_require)) egg_info.egg_info.run = replacement_run exec(compile(open(__file__).read().replace('\\r\\n', '\\n'), __file__, 'exec')) """ @@ -324,7 +379,7 @@ def trim_zeroes(version): return version -def requires_and_conflicts(req_list): +def requires_and_conflicts(req_list, skip_req_names=()): rpm_requires = "" rpm_conflicts = "" for line in req_list: @@ -332,6 +387,8 @@ def requires_and_conflicts(req_list): req = pkg_resources.Requirement.parse(line) except Exception: continue + if req.key in skip_req_names: + continue rpm_name = python_key_to_rpm(req.key) if not req.specs: rpm_requires += "\nRequires:" @@ -383,6 +440,9 @@ def build_rpm(options, filename): info = pkg_info(source_dir) rpm_requires, rpm_conflicts = requires_and_conflicts( egg_info_requirements(source_dir)) + test_rpm_requires, test_rpm_conflicts = requires_and_conflicts( + egg_info_requirements(source_dir, filename="test-requires.txt"), + skip_req_names=('sphinx', 'setuptools', 'setuptools-git', 'docutils')) # NOTE(aababilov): do not use info["name"] to get the name - it is # the key (e.g., "nose-plugin"), not the name ("nose_plugin") pkg_name = setup_py_one_line(source_dir, "--name") @@ -411,10 +471,14 @@ def build_rpm(options, filename): cleaned_version = trim_zeroes(version.replace('-', '_')) with open(spec_name, "w") as spec_file: print >> spec_file, "%define pkg_name", pkg_name + print >> spec_file, "%define pkg_path", os.path.join(*pkg_name.split('.')) print >> spec_file, "%define rpm_name", rpm_name print >> spec_file, "%define version", cleaned_version print >> spec_file, "%define release", options.release print >> spec_file, "%define unmangled_version", version + if options.with_tests: + print >> spec_file, ("%define tests_data_dir " + "%{_datarootdir}/%{pkg_name}-tests") print >> spec_file, "" tags = [] tags.append(("Name", "%{rpm_name}")) @@ -448,6 +512,21 @@ def build_rpm(options, filename): print >> spec_file, rpm_conflicts print >> spec_file, "\n%description\n", info["description"] + if options.with_tests: + print >> spec_file, "\n%package tests" + print >> spec_file, "Group: Development/Libraries" + print >> spec_file, "Summary: tests for %{name}" + for req in ("%{name} = %{epoch}:%{version}-%{release}", + "python-nose", + "python-openstack-nose-plugin", + "python-nose-exclude"): + print >> spec_file, "Requires:", req + print >> spec_file, test_rpm_requires + print >> spec_file, test_rpm_conflicts + print >> spec_file, "\n%description tests" + print >> spec_file, "Tests for %{name}" + + for script in "prep", "build", "install", "clean": print >> spec_file, "\n\n%%%s" % script if options.scripts_dir: @@ -457,11 +536,17 @@ def build_rpm(options, filename): print >> spec_file, f_in.read() continue print >> spec_file, DEFAULT_SCRIPTS[script] + if options.with_tests and script in DEFAULT_TESTS_SCRIPTS: + print >> spec_file, DEFAULT_TESTS_SCRIPTS[script] print >> spec_file, """ %files -f INSTALLED_FILES %defattr(-,root,root) """ + if options.with_tests: + print >> spec_file, "\n%files tests" + print >> spec_file, "%{_bindir}/%{pkg_name}-make-test-env" + print >> spec_file, "%{tests_data_dir}" if options.source_only: rpmbuild_what = "-bs"