From a17570110f399139919a3d0f98b6e63f78d733f1 Mon Sep 17 00:00:00 2001 From: Gabriel Hurley Date: Wed, 4 Jan 2012 14:46:40 -0800 Subject: [PATCH] Unified horizon and openstack-dashboard environments. Buildout has been removed entirely, all dev installation is now done via the single tools/install_venv.py script. Django's manage.py script has also been updated to a newer version/convention; this allows for less python path mangling and makes things cleaner and more explicit. Note that, as such, it has been moved up a directory level. Change-Id: I62f9f06ee00568fc91e5ba7e1fd15d22ea849d1f --- .gitignore | 13 +- README.rst | 18 +- docs/source/quickstart.rst | 23 +- docs/source/ref/run_tests.rst | 11 +- horizon/Makefile | 7 - horizon/README | 41 +-- horizon/bootstrap.py | 260 ------------------ horizon/buildout.cfg | 123 --------- openstack-dashboard/Makefile | 7 - openstack-dashboard/README | 14 +- openstack-dashboard/dashboard/manage.py | 38 --- openstack-dashboard/dashboard/settings.py | 3 +- openstack-dashboard/manage.py | 9 + run_tests.sh | 194 ++++--------- .../tools => tools}/install_venv.py | 8 +- .../tools => tools}/pip-requires | 0 .../tools => tools}/with_venv.sh | 2 +- 17 files changed, 116 insertions(+), 655 deletions(-) delete mode 100644 horizon/bootstrap.py delete mode 100755 horizon/buildout.cfg delete mode 100755 openstack-dashboard/dashboard/manage.py create mode 100644 openstack-dashboard/manage.py rename {openstack-dashboard/tools => tools}/install_venv.py (96%) rename {openstack-dashboard/tools => tools}/pip-requires (100%) rename {openstack-dashboard/tools => tools}/with_venv.sh (67%) diff --git a/.gitignore b/.gitignore index 376ccea28..45f547d88 100644 --- a/.gitignore +++ b/.gitignore @@ -8,19 +8,8 @@ coverage.xml pep8.txt pylint.txt reports -horizon/.installed.cfg -horizon/bin -horizon/develop-eggs/ -horizon/downloads/ -horizon/eggs/ -horizon/htmlcov -horizon/launchpad -horizon/parts/ -horizon/django_nova.egg-info horizon/horizon.egg-info -horizon/django_openstack.egg-info -django-nova-syspanel/src/django_nova_syspanel.egg-info -openstack-dashboard/.dashboard-venv +.horizon-venv openstack-dashboard/local/dashboard_openstack.sqlite3 openstack-dashboard/local/local_settings.py docs/build/ diff --git a/README.rst b/README.rst index a5c3acb5e..804bcc48e 100644 --- a/README.rst +++ b/README.rst @@ -34,18 +34,22 @@ two very distinct components underneath it: ``horizon``, and ``openstack-dashboard``. The ``horizon`` directory holds the generic libraries and components that can -be used in any Django project. In testing, this component is set up with -buildout (see ``run_tests.sh``), and any dependencies that get added need to -be added to the ``horizon/buildout.cfg`` file. +be used in any Django project. The ``openstack-dashboard`` directory contains a reference Django project that -uses ``horizon`` and is built with a virtualenv and tested through that -environment. If dependencies are added that ``openstack-dashboard`` requires -they should be added to ``openstack-dashboard/tools/pip-requires``. +uses ``horizon``. + +For development, both pieces share an environment which (by default) is +built with the ``tools/install_venv.py`` script. That script creates a +virtualenv and installs all the necessary packages. + +If dependencies are added to either ``horizon`` or ``openstack-dashboard``, +they should be added to ``tools/pip-requires``. The ``run_tests.sh`` script invokes tests and analyses on both of these components in its process, and is what Jenkins uses to verify the -stability of the project. +stability of the project. If run before an environment is set up, it will +ask if you wish to install one. To run the tests:: diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index 689556596..4cd33c86b 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -34,21 +34,22 @@ an OpenStack development environment from scratch. Horizon's Structure =================== -This project is a bit different from other Openstack projects in that it is -composed of two distinct components: - - * ``horizon`` - * ``openstack-dashboard`` +This project is a bit different from other OpenStack projects in that it has +two very distinct components underneath it: ``horizon``, and +``openstack-dashboard``. The ``horizon`` directory holds the generic libraries and components that can -be used in any Django project. In testing, this component is set up with -buildout (see :doc:`ref/run_tests`), and any dependencies that need to -be added to the ``horizon/buildout.cfg`` file. +be used in any Django project. The ``openstack-dashboard`` directory contains a reference Django project that -uses ``horizon`` and is built with a virtualenv. If dependencies are added that -``openstack-dashboard`` requires they should be added to ``openstack- -dashboard/tools/pip-requires``. +uses ``horizon``. + +For development, both pieces share an environment which (by default) is +built with the ``tools/install_venv.py`` script. That script creates a +virtualenv and installs all the necessary packages. + +If dependencies are added to either ``horizon`` or ``openstack-dashboard``, +they should be added to ``tools/pip-requires``. .. important:: diff --git a/docs/source/ref/run_tests.rst b/docs/source/ref/run_tests.rst index 98e878fee..7c1f5a8ee 100644 --- a/docs/source/ref/run_tests.rst +++ b/docs/source/ref/run_tests.rst @@ -15,18 +15,17 @@ First Run If you start with a clean copy of the Horizon repository, the first thing you should do is to run ``./run_tests.sh`` from the root of the repository. -This will do three things for you: +This will do two things for you: - #. Set up a virtual environment for ``openstack-dashboard`` using + #. Set up a virtual environment for both the ``horizon`` module and + the ``openstack-dashboard`` project using ``openstack-dashboard/tools/install_venv.py``. - #. Set up an environment for ``horizon`` using - ``horizon/bootstrap.py`` and ``horizon/bin/buildout``. #. Run the tests for both ``horizon`` and ``openstack-dashboard`` using their respective environments and verify that evreything is working. -Setting up both environments the first time can take several minutes, but only +Setting up the environment the first time can take several minutes, but only needs to be done once. If dependencies are added in the future, updating the -environments will be necessary but not necessarily as time consuming. +environments will be necessary but not as time consuming. I just want to run the tests! ============================= diff --git a/horizon/Makefile b/horizon/Makefile index 19b180fce..49e4f7288 100644 --- a/horizon/Makefile +++ b/horizon/Makefile @@ -3,19 +3,12 @@ DESTDIR=/ PROJECT=horizon all: - @echo "make buildout - Run through buildout" @echo "make test - Run tests" @echo "make source - Create source package" @echo "make install - Install on local system" @echo "make buildrpm - Generate a rpm package" @echo "make clean - Get rid of scratch and byte files" -buildout: ./bin/buildout - ./bin/buildout - -./bin/buildout: - $(PYTHON) bootstrap.py - source: $(PYTHON) setup.py sdist $(COMPILE) diff --git a/horizon/README b/horizon/README index 1b6d6fd33..76f9607be 100644 --- a/horizon/README +++ b/horizon/README @@ -21,39 +21,10 @@ you can disregard this advice. Getting Started =============== -Horizon uses Buildout (http://www.buildout.org/) to manage local development. -To configure your local Buildout environment first install the following -system-level dependencies: +Horizon uses the common environment configured by ``tools/install_venv.py`` +based on the dependencies listed in ``tools/pip-requires`` for local +development. - * python-dev - * git - * bzr - -Then instantiate buildout with:: - - $ python bootstrap.py - $ bin/buildout - -This will install all the dependencies of Horizon and provide some useful -scripts in the ``bin/`` directory: - - bin/python provides a python shell for the current buildout. - bin/django provides django functions for the current buildout. - - -You should now be able to run unit tests as follows:: - - $ bin/django test - -or:: - - $ bin/test - -You can run unit tests with code coverage on Horizon by setting -``NOSE_WITH_COVERAGE``:: - - $ NOSE_WITH_COVERAGE=true bin/test - -Get even better coverage info by running coverage directly:: - - $ coverage run --branch --source horizon bin/django test horizon && coverage html +The simplest way to get going is to simply run the ``run_tests.sh`` script +included with the Horizon project. This will set up your local environment +and run the full test suite to verify that everything is working properly. diff --git a/horizon/bootstrap.py b/horizon/bootstrap.py deleted file mode 100644 index 5f2cb0835..000000000 --- a/horizon/bootstrap.py +++ /dev/null @@ -1,260 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os, shutil, sys, tempfile, textwrap, urllib, urllib2, subprocess -from optparse import OptionParser - -if sys.platform == 'win32': - def quote(c): - if ' ' in c: - return '"%s"' % c # work around spawn lamosity on windows - else: - return c -else: - quote = str - -# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments. -stdout, stderr = subprocess.Popen( - [sys.executable, '-Sc', - 'try:\n' - ' import ConfigParser\n' - 'except ImportError:\n' - ' print 1\n' - 'else:\n' - ' print 0\n'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() -has_broken_dash_S = bool(int(stdout.strip())) - -# In order to be more robust in the face of system Pythons, we want to -# run without site-packages loaded. This is somewhat tricky, in -# particular because Python 2.6's distutils imports site, so starting -# with the -S flag is not sufficient. However, we'll start with that: -if not has_broken_dash_S and 'site' in sys.modules: - # We will restart with python -S. - args = sys.argv[:] - args[0:0] = [sys.executable, '-S'] - args = map(quote, args) - os.execv(sys.executable, args) -# Now we are running with -S. We'll get the clean sys.path, import site -# because distutils will do it later, and then reset the path and clean -# out any namespace packages from site-packages that might have been -# loaded by .pth files. -clean_path = sys.path[:] -import site -sys.path[:] = clean_path -for k, v in sys.modules.items(): - if k in ('setuptools', 'pkg_resources') or ( - hasattr(v, '__path__') and - len(v.__path__)==1 and - not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))): - # This is a namespace package. Remove it. - sys.modules.pop(k) - -is_jython = sys.platform.startswith('java') - -setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py' -distribute_source = 'http://python-distribute.org/distribute_setup.py' - -# parsing arguments -def normalize_to_url(option, opt_str, value, parser): - if value: - if '://' not in value: # It doesn't smell like a URL. - value = 'file://%s' % ( - urllib.pathname2url( - os.path.abspath(os.path.expanduser(value))),) - if opt_str == '--download-base' and not value.endswith('/'): - # Download base needs a trailing slash to make the world happy. - value += '/' - else: - value = None - name = opt_str[2:].replace('-', '_') - setattr(parser.values, name, value) - -usage = '''\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --setup-source and --download-base to point to -local resources, you can keep this script from going over the network. -''' - -parser = OptionParser(usage=usage) -parser.add_option("-v", "--version", dest="version", - help="use a specific zc.buildout version") -parser.add_option("-d", "--distribute", - action="store_true", dest="use_distribute", default=False, - help="Use Distribute rather than Setuptools.") -parser.add_option("--setup-source", action="callback", dest="setup_source", - callback=normalize_to_url, nargs=1, type="string", - help=("Specify a URL or file location for the setup file. " - "If you use Setuptools, this will default to " + - setuptools_source + "; if you use Distribute, this " - "will default to " + distribute_source +".")) -parser.add_option("--download-base", action="callback", dest="download_base", - callback=normalize_to_url, nargs=1, type="string", - help=("Specify a URL or directory for downloading " - "zc.buildout and either Setuptools or Distribute. " - "Defaults to PyPI.")) -parser.add_option("--eggs", - help=("Specify a directory for storing eggs. Defaults to " - "a temporary directory that is deleted when the " - "bootstrap script completes.")) -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", None, action="store", dest="config_file", - help=("Specify the path to the buildout configuration " - "file to be used.")) - -options, args = parser.parse_args() - -# if -c was provided, we push it back into args for buildout's main function -if options.config_file is not None: - args += ['-c', options.config_file] - -if options.eggs: - eggs_dir = os.path.abspath(os.path.expanduser(options.eggs)) -else: - eggs_dir = tempfile.mkdtemp() - -if options.setup_source is None: - if options.use_distribute: - options.setup_source = distribute_source - else: - options.setup_source = setuptools_source - -if options.accept_buildout_test_releases: - args.append('buildout:accept-buildout-test-releases=true') -args.append('bootstrap') - -try: - import pkg_resources - import setuptools # A flag. Sometimes pkg_resources is installed alone. - if not hasattr(pkg_resources, '_distribute'): - raise ImportError -except ImportError: - ez_code = urllib2.urlopen( - options.setup_source).read().replace('\r\n', '\n') - ez = {} - exec ez_code in ez - setup_args = dict(to_dir=eggs_dir, download_delay=0) - if options.download_base: - setup_args['download_base'] = options.download_base - if options.use_distribute: - setup_args['no_fake'] = True - ez['use_setuptools'](**setup_args) - if 'pkg_resources' in sys.modules: - reload(sys.modules['pkg_resources']) - import pkg_resources - # This does not (always?) update the default working set. We will - # do it. - for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -cmd = [quote(sys.executable), - '-c', - quote('from setuptools.command.easy_install import main; main()'), - '-mqNxd', - quote(eggs_dir)] - -if not has_broken_dash_S: - cmd.insert(1, '-S') - -find_links = options.download_base -if not find_links: - find_links = os.environ.get('bootstrap-testing-find-links') -if find_links: - cmd.extend(['-f', quote(find_links)]) - -if options.use_distribute: - setup_requirement = 'distribute' -else: - setup_requirement = 'setuptools' -ws = pkg_resources.working_set -setup_requirement_path = ws.find( - pkg_resources.Requirement.parse(setup_requirement)).location -env = dict( - os.environ, - PYTHONPATH=setup_requirement_path) - -requirement = 'zc.buildout' -version = options.version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - _final_parts = '*final-', '*final' - def _final_version(parsed_version): - for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): - return False - return True - index = setuptools.package_index.PackageIndex( - search_path=[setup_requirement_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = '=='.join((requirement, version)) -cmd.append(requirement) - -if is_jython: - import subprocess - exitcode = subprocess.Popen(cmd, env=env).wait() -else: # Windows prefers this, apparently; otherwise we would prefer subprocess - exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env])) -if exitcode != 0: - sys.stdout.flush() - sys.stderr.flush() - print ("An error occurred when trying to install zc.buildout. " - "Look above this message for any errors that " - "were output by easy_install.") - sys.exit(exitcode) - -ws.add_entry(eggs_dir) -ws.require(requirement) -import zc.buildout.buildout -zc.buildout.buildout.main(args) -if not options.eggs: # clean up temporary egg directory - shutil.rmtree(eggs_dir) diff --git a/horizon/buildout.cfg b/horizon/buildout.cfg deleted file mode 100755 index 3eaf99f41..000000000 --- a/horizon/buildout.cfg +++ /dev/null @@ -1,123 +0,0 @@ -[buildout] -download-cache = /tmp/.buildout_cache/ -parts = - django - openstackx - glance - quantum - python-novaclient - python-keystoneclient - seleniumrc -develop = . -versions = versions - - -[versions] -django = 1.3.1 -# the following are for glance-dependencies -greenlet = 0.3.1 -eventlet = 0.9.12 -pep8 = 0.5.0 -sqlalchemy = 0.6.3 -sqlalchemy-migrate = 0.6 -webob = 1.0.8 -pycrypto = 2.3 - - -[dependencies] -# dependencies that are found locally ${buildout:directory}/module -# or can be fetched from pypi -recipe = zc.recipe.egg -eggs = - python-dateutil - httplib2 - python-cloudfiles - coverage - django-nose-selenium - CherryPy - pycrypto -interpreter = python - - -# glance doesn't properly list it's dependencies, so we have to install them -[glance-dependencies] -recipe = zc.recipe.egg -eggs = - PasteDeploy - anyjson - argparse - eventlet - greenlet - kombu - paste - pep8 - routes - sqlalchemy - sqlalchemy-migrate - webob - xattr -interpreter = python - - -[horizon] -recipe = zc.recipe.egg -eggs = horizon -interpreter = python - - -[django] -# defines settings for django -# any dependencies that cannot be satisifed via the dependencies -# recipe above will need to be added to the extra-paths here. -# IE, dependencies fetch from a git repo will not auto-populate -# like the zc.recipe.egg ones will -recipe = djangorecipe -project = horizon -projectegg = horizon -settings = tests -test = horizon -eggs = - ${dependencies:eggs} - ${horizon:eggs} - ${glance-dependencies:eggs} -extra-paths = - ${buildout:directory}/parts/openstackx - ${buildout:directory}/parts/python-novaclient - ${buildout:directory}/parts/python-keystoneclient - - -## Dependencies fetch from git -# git dependencies end up as a subdirectory of ${buildout:directory}/parts/ -[openstackx] -recipe = zerokspot.recipe.git -repository = git://github.com/cloudbuilders/openstackx.git -as_egg = True - - -[glance] -recipe = zerokspot.recipe.git -repository = git://github.com/openstack/glance.git -as_egg = True - - -[quantum] -recipe = zerokspot.recipe.git -repository = git://github.com/openstack/quantum.git -branch = stable/diablo -as_egg = True - - -[python-novaclient] -recipe = zerokspot.recipe.git -repository = git://github.com/openstack/python-novaclient.git -as_egg = True - - -[python-keystoneclient] -recipe = zerokspot.recipe.git -repository = git://github.com/openstack/python-keystoneclient.git -as_egg = True - - -[seleniumrc] -recipe=collective.recipe.seleniumrc diff --git a/openstack-dashboard/Makefile b/openstack-dashboard/Makefile index 63df40114..39a309eb4 100644 --- a/openstack-dashboard/Makefile +++ b/openstack-dashboard/Makefile @@ -3,19 +3,12 @@ DESTDIR=/ PROJECT=openstack-dashboard all: - @echo "make buildout - Run through buildout" @echo "make test - Run tests" @echo "make source - Create source package" @echo "make install - Install on local system" @echo "make buildrpm - Generate a rpm package" @echo "make clean - Get rid of scratch and byte files" -buildout: ./bin/buildout - ./bin/buildout - -./bin/buildout: - $(PYTHON) bootstrap.py - source: $(PYTHON) setup.py sdist $(COMPILE) diff --git a/openstack-dashboard/README b/openstack-dashboard/README index f902e97db..276bc9e0b 100644 --- a/openstack-dashboard/README +++ b/openstack-dashboard/README @@ -11,18 +11,26 @@ Getting Started =============== For local development, first create a virtualenv for the project. -A tool is included to create one for you: +In the Horizon ``tools`` directory there is a script to create one for you: $ python tools/install_venv.py +Alternatively, the ``run_tests.sh`` script will also install the environment +for you and then run the full test suite to verify everything is installed +and functioning correctly. + Now that the virtualenv is created, you need to configure your local environment. To do this, create a ``local_settings.py`` file in the ``local/`` directory. There is a ``local_settings.py.example`` file there that may be used as a template. -If all is well you should now able to run the server locally: +If all is well you should now able to run the development server locally: - $ tools/with_venv.sh dashboard/manage.py runserver + $ tools/with_venv.sh openstack-dashboard/manage.py runserver + +or, as a shortcut:: + + $ ./run_tests.sh --runserver Settings Up OpenStack ===================== diff --git a/openstack-dashboard/dashboard/manage.py b/openstack-dashboard/dashboard/manage.py deleted file mode 100755 index 73053f395..000000000 --- a/openstack-dashboard/dashboard/manage.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Copyright 2011 Nebula, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from django.core.management import execute_manager - - -try: - import settings # Assumed to be in the same directory. -except ImportError: - import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the " - "directory containing %r. It appears you've customized things.\nYou'll " - "have to run django-admin.py, passing it your settings module.\n(If " - "the file settings.py does indeed exist, it's causing an ImportError " - "somehow.)\n" % __file__) - sys.exit(1) - - -if __name__ == "__main__": - execute_manager(settings) diff --git a/openstack-dashboard/dashboard/settings.py b/openstack-dashboard/dashboard/settings.py index ad9de3359..c76407b73 100644 --- a/openstack-dashboard/dashboard/settings.py +++ b/openstack-dashboard/dashboard/settings.py @@ -24,7 +24,8 @@ import sys ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(ROOT_PATH) +if ROOT_PATH not in sys.path: + sys.path.append(ROOT_PATH) DEBUG = False TEMPLATE_DEBUG = DEBUG diff --git a/openstack-dashboard/manage.py b/openstack-dashboard/manage.py new file mode 100644 index 000000000..ca1950470 --- /dev/null +++ b/openstack-dashboard/manage.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +import os, sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/run_tests.sh b/run_tests.sh index da02b1a72..f15c3ee20 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -6,7 +6,7 @@ set -o errexit # Increment me any time the environment should be rebuilt. # This includes dependncy changes, directory renames, etc. # Simple integer secuence: 1, 2, 3... -environment_version=6 +environment_version=7 #--------------------------------------------------------# function usage { @@ -45,26 +45,27 @@ function usage { # DEFAULTS FOR RUN_TESTS.SH # -venv=openstack-dashboard/.dashboard-venv -django_with_venv=openstack-dashboard/tools/with_venv.sh -dashboard_with_venv=tools/with_venv.sh +root=`pwd` +venv=$root/.horizon-venv +with_venv=tools/with_venv.sh +included_dirs="openstack-dashboard/dashboard horizon/horizon" + always_venv=0 -never_venv=0 +backup_env=0 +command_wrapper="" +destroy=0 force=0 -with_coverage=0 -selenium=0 -testargs="" -django_wrapper="" -dashboard_wrapper="" just_pep8=0 just_pylint=0 just_docs=0 just_tabs=0 -runserver=0 +never_venv=0 quiet=0 -backup_env=0 restore_env=0 -destroy=0 +runserver=0 +selenium=0 +testargs="" +with_coverage=0 # Jenkins sets a "JOB_NAME" variable, if it's not set, we'll make it "default" [ "$JOB_NAME" ] || JOB_NAME="default" @@ -92,18 +93,16 @@ function process_option { function run_server { echo "Starting Django development server..." - ${django_wrapper} python openstack-dashboard/dashboard/manage.py runserver $testargs + ${command_wrapper} python $root/openstack-dashboard/manage.py runserver $testargs echo "Server stopped." } function run_pylint { echo "Running pylint ..." - PYLINT_INCLUDE="openstack-dashboard/dashboard horizon/horizon" - ${django_wrapper} pylint --rcfile=.pylintrc -f parseable $PYLINT_INCLUDE > pylint.txt || true + PYTHONPATH=$root/openstack-dashboard ${command_wrapper} pylint --rcfile=.pylintrc -f parseable $included_dirs > pylint.txt || true CODE=$? grep Global -A2 pylint.txt - if [ $CODE -lt 32 ] - then + if [ $CODE -lt 32 ]; then echo "Completed successfully." exit 0 else @@ -117,9 +116,7 @@ function run_pep8 { rm -f pep8.txt PEP8_EXCLUDE=vcsversion.py PEP8_OPTIONS="--exclude=$PEP8_EXCLUDE --repeat" - PEP8_INCLUDE="openstack-dashboard/dashboard horizon/horizon" - echo "${django_wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE > pep8.txt" - ${django_wrapper} pep8 $PEP8_OPTIONS $PEP8_INCLUDE | perl -ple 's/: ([WE]\d+)/: [$1]/' > pep8.txt || true + ${command_wrapper} pep8 $PEP8_OPTIONS $included_dirs | perl -ple 's/: ([WE]\d+)/: [$1]/' > pep8.txt || true PEP8_COUNT=`wc -l pep8.txt | awk '{ print $1 }'` if [ $PEP8_COUNT -ge 1 ]; then echo "PEP8 violations found ($PEP8_COUNT):" @@ -132,18 +129,16 @@ function run_pep8 { function run_sphinx { echo "Building sphinx..." - echo "export DJANGO_SETTINGS_MODULE=dashboard.settings" export DJANGO_SETTINGS_MODULE=dashboard.settings - echo "${django_wrapper} sphinx-build -b html docs/source docs/build/html" - ${django_wrapper} sphinx-build -b html docs/source docs/build/html + ${command_wrapper} sphinx-build -b html docs/source docs/build/html echo "Build complete." } function tab_check { - TAB_VIOLATIONS=`find horizon/horizon openstack-dashboard/dashboard -type f -regex ".*\.\(css\|js\|py\|html\)" -print0 | xargs -0 awk '/\t/' | wc -l` + TAB_VIOLATIONS=`find $included_dirs -type f -regex ".*\.\(css\|js\|py\|html\)" -print0 | xargs -0 awk '/\t/' | wc -l` if [ $TAB_VIOLATIONS -gt 0 ]; then echo "TABS! $TAB_VIOLATIONS of them! Oh no!" - HORIZON_FILES=`find horizon/horizon openstack-dashboard/dashboard -type f -regex ".*\.\(css\|js\|py|\html\)"` + HORIZON_FILES=`find $included_dirs -type f -regex ".*\.\(css\|js\|py|\html\)"` for TABBED_FILE in $HORIZON_FILES do TAB_COUNT=`awk '/\t/' $TABBED_FILE | wc -l` @@ -155,21 +150,10 @@ function tab_check { return $TAB_VIOLATIONS; } -function destroy_buildout { - echo "Removing buildout files..." - rm -rf horizon/bin - rm -rf horizon/eggs - rm -rf horizon/parts - rm -rf horizon/develop-eggs - rm -rf horizon/horizon.egg-info - echo "Buildout files removed." -} - function destroy_venv { - echo "Cleaning virtualenv..." - destroy_buildout + echo "Cleaning environment..." echo "Removing virtualenv..." - rm -rf openstack-dashboard/.dashboard-venv + rm -rf $venv echo "Virtualenv removed." rm -f .environment_version echo "Environment cleaned." @@ -182,8 +166,7 @@ function environment_check { if [ $ENV_VERS -eq $environment_version ]; then if [ -e ${venv} ]; then # If the environment exists and is up-to-date then set our variables - django_wrapper="${django_with_venv}" - dashboard_wrapper="${dashboard_with_venv}" + command_wrapper="${root}/${with_venv}" echo "Environment is up to date." return 0 fi @@ -191,7 +174,6 @@ function environment_check { fi if [ $always_venv -eq 1 ]; then - destroy_buildout install_venv else if [ ! -e ${venv} ]; then @@ -201,8 +183,6 @@ function environment_check { fi read update_env if [ "x$update_env" = "xY" -o "x$update_env" = "x" -o "x$update_env" = "xy" ]; then - # Buildout doesn't play nice with upgrading everytime; kill it to be safe - destroy_buildout install_venv fi fi @@ -213,20 +193,15 @@ function sanity_check { # Don't sanity-check anything environment-related in -N flag is set if [ $never_venv -eq 0 ]; then if [ ! -e ${venv} ]; then - echo "Virtualenv not found at openstack-dashboard/.dashboard-venv. Did install_venv.py succeed?" + echo "Virtualenv not found at $venv. Did install_venv.py succeed?" exit 1 fi - if [ ! -f horizon/bin/test ]; then - echo "Error: Test script not found at horizon/bin/test. Did buildout succeed?" - exit 1 - fi - if [ ! -f horizon/bin/coverage ]; then - echo "Error: Coverage script not found at horizon/bin/coverage. Did buildout succeed?" - exit 1 - fi - if [ ! -f horizon/bin/seleniumrc ]; then - echo "Error: Selenium script not found at horizon/bin/seleniumrc. Did buildout succeed?" - exit 1 + fi + if [ $selenium -eq 1 ]; then + SELENIUM_JOB=`ps -elf | grep "selenium" | grep -v grep` + if [ $? -eq 0 ]; then + echo "WARNING: Selenium doesn't appear to be running. Please start a selenium server process." + selenium=0 fi fi } @@ -243,12 +218,7 @@ function backup_environment { rm -rf /tmp/.horizon_environment/$JOB_NAME fi mkdir -p /tmp/.horizon_environment/$JOB_NAME - cp -r openstack-dashboard/.dashboard-venv /tmp/.horizon_environment/$JOB_NAME/ - cp -r horizon/bin /tmp/.horizon_environment/$JOB_NAME/ - cp -r horizon/eggs /tmp/.horizon_environment/$JOB_NAME/ - cp -r horizon/parts /tmp/.horizon_environment/$JOB_NAME/ - cp -r horizon/develop-eggs /tmp/.horizon_environment/$JOB_NAME/ - cp -r horizon/horizon.egg-info /tmp/.horizon_environment/$JOB_NAME/ + cp -r $venv /tmp/.horizon_environment/$JOB_NAME/ cp .environment_version /tmp/.horizon_environment/$JOB_NAME/ # Remove the backup now that we've completed successfully rm -rf /tmp/.horizon_environment/$JOB_NAME.old @@ -264,15 +234,8 @@ function restore_environment { return 0 fi - destroy_buildout - - cp -r /tmp/.horizon_environment/$JOB_NAME/.dashboard-venv openstack-dashboard/ - cp -r /tmp/.horizon_environment/$JOB_NAME/bin horizon/ - cp -r /tmp/.horizon_environment/$JOB_NAME/eggs horizon/ - cp -r /tmp/.horizon_environment/$JOB_NAME/parts horizon/ - cp -r /tmp/.horizon_environment/$JOB_NAME/develop-eggs horizon/ - cp -r /tmp/.horizon_environment/$JOB_NAME/horizon.egg-info horizon/ - cp -r /tmp/.horizon_environment/$JOB_NAME/.environment_version ./ + cp -r /tmp/.horizon_environment/$JOB_NAME/.horizon-venv ./ || true + cp -r /tmp/.horizon_environment/$JOB_NAME/.environment_version ./ || true echo "Environment restored successfully." fi @@ -285,111 +248,62 @@ function install_venv { if [ $quiet -eq 1 ]; then export PIP_NO_INPUT=true fi - cd openstack-dashboard INSTALL_FAILED=0 python tools/install_venv.py || INSTALL_FAILED=1 if [ $INSTALL_FAILED -eq 1 ]; then echo "Error updating environment with pip, trying without src packages..." - rm -rf .dashboard-venv/src + rm -rf $venv/src python tools/install_venv.py fi - cd .. - # Install horizon with buildout - if [ ! -d /tmp/.buildout_cache ]; then - mkdir -p /tmp/.buildout_cache - fi - cd horizon - python bootstrap.py - bin/buildout - cd .. - django_wrapper="${django_with_venv}" - dashboard_wrapper="${dashboard_with_venv}" + command_wrapper="$root/${with_venv}" # Make sure it worked and record the environment version sanity_check - chmod -R 754 openstack-dashboard/.dashboard-venv + chmod -R 754 $venv echo $environment_version > .environment_version } -function wait_for_selenium { - # Selenium can sometimes take several seconds to start. - STARTED=`grep -irn "Started SocketListener on 0.0.0.0:4444" .selenium_log` - if [ $? -eq 0 ]; then - echo "Selenium server started." - return 0 - fi - echo -n "." - sleep 1 - wait_for_selenium -} - -function stop_selenium { - if [ $selenium -eq 1 ]; then - echo "Stopping Selenium server..." - SELENIUM_JOB=`ps -elf | grep "seleniumrc" | grep -v grep` - if [ $? -eq 0 ]; then - kill `echo "${SELENIUM_JOB}" | awk '{print $4}'` - echo "Selenium process stopped." - else - echo "No selenium process running." - fi - rm -f .selenium_log - fi -} - function run_tests { sanity_check - if [ $selenium -eq 1 ]; then - stop_selenium - echo "Starting Selenium server..." - rm -f .selenium_log - ${django_wrapper} horizon/bin/seleniumrc > .selenium_log & - wait_for_selenium - fi - echo "Running Horizon application tests" - ${django_wrapper} coverage erase - ${django_wrapper} coverage run horizon/bin/test + ${command_wrapper} coverage erase + ${command_wrapper} coverage run $root/openstack-dashboard/manage.py test horizon --settings=horizon.tests.testsettings # get results of the Horizon tests - OPENSTACK_RESULT=$? + HORIZON_RESULT=$? echo "Running openstack-dashboard (Django project) tests" - cd openstack-dashboard - if [ -f local/local_settings.py ]; then - cp local/local_settings.py local/local_settings.py.bak + if [ -f $root/openstack-dashboard/local/local_settings.py ]; then + cp $root/openstack-dashboard/local/local_settings.py $root/openstack-dashboard/local/local_settings.py.bak fi - cp local/local_settings.py.example local/local_settings.py + cp $root/openstack-dashboard/local/local_settings.py.example $root/openstack-dashboard/local/local_settings.py if [ $selenium -eq 1 ]; then - ${dashboard_wrapper} coverage run dashboard/manage.py test --with-selenium --with-cherrypyliveserver + ${command_wrapper} coverage run $root/openstack-dashboard/manage.py test dashboard --with-selenium --with-cherrypyliveserver else - ${dashboard_wrapper} coverage run dashboard/manage.py test + ${command_wrapper} coverage run $root/openstack-dashboard/manage.py test dashboard fi # get results of the openstack-dashboard tests DASHBOARD_RESULT=$? - if [ -f local/local_settings.py.bak ]; then - cp local/local_settings.py.bak local/local_settings.py - rm local/local_settings.py.bak + if [ -f $root/openstack-dashboard/local/local_settings.py.bak ]; then + cp $root/openstack-dashboard/local/local_settings.py.bak $root/openstack-dashboard/local/local_settings.py + rm $root/openstack-dashboard/local/local_settings.py.bak fi - rm local/local_settings.pyc - cd .. + rm -f $root/openstack-dashboard/local/local_settings.pyc if [ $with_coverage -eq 1 ]; then echo "Generating coverage reports" - ${django_wrapper} coverage combine - ${django_wrapper} coverage xml -i --omit='/usr*,setup.py,*egg*' - ${django_wrapper} coverage html -i --omit='/usr*,setup.py,*egg*' -d reports + ${command_wrapper} coverage combine + ${command_wrapper} coverage xml -i --omit='/usr*,setup.py,*egg*' + ${command_wrapper} coverage html -i --omit='/usr*,setup.py,*egg*' -d reports fi - stop_selenium - - if [ $(($OPENSTACK_RESULT || $DASHBOARD_RESULT)) -eq 0 ]; then + if [ $(($HORIZON_RESULT || $DASHBOARD_RESULT)) -eq 0 ]; then echo "Tests completed successfully." else echo "Tests failed." fi - exit $(($OPENSTACK_RESULT || $DASHBOARD_RESULT)) + exit $(($HORIZON_RESULT || $DASHBOARD_RESULT)) } diff --git a/openstack-dashboard/tools/install_venv.py b/tools/install_venv.py similarity index 96% rename from openstack-dashboard/tools/install_venv.py rename to tools/install_venv.py index 7d3d8271f..4d4054e30 100644 --- a/openstack-dashboard/tools/install_venv.py +++ b/tools/install_venv.py @@ -30,7 +30,7 @@ import sys ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -VENV = os.path.join(ROOT, '.dashboard-venv') +VENV = os.path.join(ROOT, '.horizon-venv') WITH_VENV = os.path.join(ROOT, 'tools', 'with_venv.sh') PIP_REQUIRES = os.path.join(ROOT, 'tools', 'pip-requires') @@ -121,18 +121,18 @@ def install_dependencies(venv=VENV): def install_django_openstack(): print 'Installing horizon module in development mode...' - path = os.path.join(ROOT, '..', 'horizon') + path = os.path.join(ROOT, 'horizon') run_command([WITH_VENV, 'python', 'setup.py', 'develop'], cwd=path) def print_summary(): summary = """ - OpenStack Dashboard development environment setup is complete. + Horizon development environment setup is complete. To activate the virtualenv for the extent of your current shell session you can run: - $ source .dashboard-venv/bin/activate + $ source .horizon-venv/bin/activate """ print summary diff --git a/openstack-dashboard/tools/pip-requires b/tools/pip-requires similarity index 100% rename from openstack-dashboard/tools/pip-requires rename to tools/pip-requires diff --git a/openstack-dashboard/tools/with_venv.sh b/tools/with_venv.sh similarity index 67% rename from openstack-dashboard/tools/with_venv.sh rename to tools/with_venv.sh index 51efe29fd..59d915f91 100755 --- a/openstack-dashboard/tools/with_venv.sh +++ b/tools/with_venv.sh @@ -1,4 +1,4 @@ #!/bin/bash TOOLS=`dirname $0` -VENV=$TOOLS/../.dashboard-venv +VENV=$TOOLS/../.horizon-venv source $VENV/bin/activate && $@