openstack/swift
Code Issues Proposed changes

Make in process functional tests use sample proxy-server.conf

Browse Source 
This patch was first motivated by noticing that the proxy
server pipeline used for in process functional tests was
out of date with respect to the pipeline in
/etc/proxy-server.conf.sample. Rather than cut and paste
the current pipeline into the in process setup, it seems
like a better idea would be to have the in process tests
always use the sample config.

A further benefit is that in process functional tests will
pick up changes to the sample config introduced by patches -
previously test/functional/__init__.py would need to be
manually modified to run in process functional tests
on new middleware for example.

Note: because the pipeline is now loaded using entry points,
'python setup.py [develop|install]' will now be needed
before running the tests.

Obvious next steps would be to do the same for the backend
servers, and to allow alternative config files and dir's
to be specified, but this patch is the first step.

Also drive-by fixes some typos in proxy-server.conf.sample

Change-Id: If442bd7c2b1721ec92839c4490924ba33e1545d8
This commit is contained in:
Alistair Coles 2014-08-11 17:09:48 +01:00
parent 75cca49334
commit c9f8246378
2 changed files with 40 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/proxy-server.conf-sample
View File

@ -17,7 +17,7 @@ bind_port = 8080
# to /info. You can withhold subsections by separating the dict level with a
# ".". The following would cause the sections 'container_quotas' and 'tempurl'
# to not be listed, and the key max_failed_deletes would be removed from
# bulk_delete. Default is empty, allowing all registered fetures to be listed
# bulk_delete. Default is empty, allowing all registered features to be listed
# via HTTP GET /info.
# disallowed_sections = container_quotas, tempurl, bulk_delete.max_failed_deletes
@ -592,7 +592,7 @@ use = egg:swift#container_sync
# Set this to specify this clusters //realm/cluster as "current" in /info
# current = //REALM/CLUSTER
# Note: Put it at the beginning of the pipleline to profile all middleware. But
# Note: Put it at the beginning of the pipeline to profile all middleware. But
# it is safer to put this after catch_errors, gatekeeper and healthcheck.
[filter:xprofile]
use = egg:swift#xprofile

92
test/functional/__init__.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import os
import sys
import pickle
@ -30,6 +31,8 @@ from contextlib import closing
from gzip import GzipFile
from shutil import rmtree
from tempfile import mkdtemp
from swift.common.memcached import MemcacheConnectionError
from swift.common.middleware.memcache import MemcacheMiddleware
from test import get_config
from test.functional.swift_test_client import Account, Connection, \
@ -40,12 +43,9 @@ from test.functional.swift_test_client import Account, Connection, \
from test.unit import debug_logger, FakeMemcache
from swift.common import constraints, utils, ring, storage_policy
from swift.common.wsgi import monkey_patch_mimetools
from swift.common.middleware import catch_errors, gatekeeper, healthcheck, \
proxy_logging, container_sync, bulk, tempurl, slo, dlo, ratelimit, \
tempauth, container_quotas, account_quotas
from swift.common.ring import Ring
from swift.common.wsgi import monkey_patch_mimetools, loadapp
from swift.common.utils import config_true_value
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server, mem_server as mem_object_server
@ -103,24 +103,19 @@ in_process = False
_testdir = _test_servers = _test_sockets = _test_coros = None
class FakeMemcacheMiddleware(object):
class FakeMemcacheMiddleware(MemcacheMiddleware):
"""
Caching middleware that fakes out caching in swift.
Caching middleware that fakes out caching in swift if memcached
does not appear to be running.
"""
def __init__(self, app, conf):
self.app = app
self.memcache = FakeMemcache()
def __call__(self, env, start_response):
env['swift.cache'] = self.memcache
return self.app(env, start_response)
def fake_memcache_filter_factory(conf):
def filter_app(app):
return FakeMemcacheMiddleware(app, conf)
return filter_app
super(FakeMemcacheMiddleware, self).__init__(app, conf)
try:
self.memcache.incr('test_for_memcached_running')
except MemcacheConnectionError as e:
print >>sys.stderr, 'Using FakeMemcache: %s' % e.message
self.memcache = FakeMemcache()
# swift.conf contents for in-process functional test runs
@ -137,6 +132,15 @@ max_file_size = %d
def in_process_setup(the_object_server=object_server):
print >>sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS'
print >>sys.stderr, 'Using object_server: %s' % the_object_server.__name__
_dir = os.path.normpath(os.path.join(os.path.abspath(__file__),
os.pardir, os.pardir, os.pardir))
proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample')
if os.path.exists(proxy_conf):
print >>sys.stderr, 'Using proxy-server config from %s' % proxy_conf
else:
print >>sys.stderr, 'Failed to find conf file %s' % proxy_conf
return
monkey_patch_mimetools()
@ -163,7 +167,9 @@ def in_process_setup(the_object_server=object_server):
if constraints.SWIFT_CONSTRAINTS_LOADED:
# Use the swift constraints that are loaded for the test framework
# configuration
config.update(constraints.EFFECTIVE_CONSTRAINTS)
_c = dict((k, str(v))
for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
config.update(_c)
else:
# In-process swift constraints were not loaded, somethings wrong
raise SkipTest
@ -184,12 +190,9 @@ def in_process_setup(the_object_server=object_server):
'devices': _testdir,
'swift_dir': _testdir,
'mount_check': 'false',
'client_timeout': 4,
'client_timeout': '4',
'allow_account_management': 'true',
'account_autocreate': 'true',
'allowed_headers':
'content-disposition, content-encoding, x-delete-at,'
' x-object-manifest, x-static-large-object',
'allow_versions': 'True',
# Below are values used by the functional test framework, as well as
# by the various in-process swift servers
@ -261,7 +264,6 @@ def in_process_setup(the_object_server=object_server):
# Default to only 4 seconds for in-process functional test runs
eventlet.wsgi.WRITE_TIMEOUT = 4
prosrv = proxy_server.Application(config, logger=debug_logger('proxy'))
acc1srv = account_server.AccountController(
config, logger=debug_logger('acct1'))
acc2srv = account_server.AccountController(
@ -274,35 +276,16 @@ def in_process_setup(the_object_server=object_server):
config, logger=debug_logger('obj1'))
obj2srv = the_object_server.ObjectController(
config, logger=debug_logger('obj2'))
global _test_servers
_test_servers = \
(prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv)
pipeline = [
catch_errors.filter_factory,
gatekeeper.filter_factory,
healthcheck.filter_factory,
proxy_logging.filter_factory,
fake_memcache_filter_factory,
container_sync.filter_factory,
bulk.filter_factory,
tempurl.filter_factory,
slo.filter_factory,
dlo.filter_factory,
ratelimit.filter_factory,
tempauth.filter_factory,
container_quotas.filter_factory,
account_quotas.filter_factory,
proxy_logging.filter_factory,
]
app = prosrv
import mock
for filter_factory in reversed(pipeline):
app_filter = filter_factory(config)
with mock.patch('swift.common.utils') as mock_utils:
mock_utils.get_logger.return_value = None
app = app_filter(app)
app.logger = prosrv.logger
logger = debug_logger('proxy')
def get_logger(name, *args, **kwargs):
return logger
with mock.patch('swift.common.utils.get_logger', get_logger):
with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
FakeMemcacheMiddleware):
app = loadapp(proxy_conf, global_conf=config)
nl = utils.NullLogger()
prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
@ -319,7 +302,8 @@ def in_process_setup(the_object_server=object_server):
# Create accounts "test" and "test2"
def create_account(act):
ts = utils.normalize_timestamp(time())
partition, nodes = prosrv.account_ring.get_nodes(act)
account_ring = Ring(_testdir, ring_name='account')
partition, nodes = account_ring.get_nodes(act)
for node in nodes:
# Note: we are just using the http_connect method in the object
# controller here to talk to the account server nodes.