diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index e4c69778bb..5d41a1d94e 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -27,7 +27,7 @@ from zlib import compressobj from swift.common.utils import quote from swift.common.http import HTTP_NOT_FOUND from swift.common.swob import Request -from swift.common.wsgi import loadapp +from swift.common.wsgi import loadapp, pipeline_property class UnexpectedResponse(Exception): @@ -142,6 +142,12 @@ class InternalClient(object): self.user_agent = user_agent self.request_tries = request_tries + get_object_ring = pipeline_property('get_object_ring') + container_ring = pipeline_property('container_ring') + account_ring = pipeline_property('account_ring') + auto_create_account_prefix = pipeline_property( + 'auto_create_account_prefix', default='.') + def make_request( self, method, path, headers, acceptable_statuses, body_file=None): """ @@ -190,7 +196,8 @@ class InternalClient(object): raise exc_type(*exc_value.args), None, exc_traceback def _get_metadata( - self, path, metadata_prefix='', acceptable_statuses=(2,)): + self, path, metadata_prefix='', acceptable_statuses=(2,), + headers=None): """ Gets metadata by doing a HEAD on a path and using the metadata_prefix to get values from the headers returned. @@ -201,6 +208,7 @@ class InternalClient(object): keys in the dict returned. Defaults to ''. :param acceptable_statuses: List of status for valid responses, defaults to (2,). + :param headers: extra headers to send :returns : A dict of metadata with metadata_prefix stripped from keys. Keys will be lowercase. @@ -211,9 +219,8 @@ class InternalClient(object): unexpected way. """ - resp = self.make_request('HEAD', path, {}, acceptable_statuses) - if not resp.status_int // 100 == 2: - return {} + headers = headers or {} + resp = self.make_request('HEAD', path, headers, acceptable_statuses) metadata_prefix = metadata_prefix.lower() metadata = {} for k, v in resp.headers.iteritems(): @@ -544,7 +551,8 @@ class InternalClient(object): def delete_object( self, account, container, obj, - acceptable_statuses=(2, HTTP_NOT_FOUND)): + acceptable_statuses=(2, HTTP_NOT_FOUND), + headers=None): """ Deletes an object. @@ -553,6 +561,7 @@ class InternalClient(object): :param obj: The object. :param acceptable_statuses: List of status for valid responses, defaults to (2, HTTP_NOT_FOUND). + :param headers: extra headers to send with request :raises UnexpectedResponse: Exception raised when requests fail to get a response with an acceptable status @@ -561,11 +570,11 @@ class InternalClient(object): """ path = self.make_path(account, container, obj) - self.make_request('DELETE', path, {}, acceptable_statuses) + self.make_request('DELETE', path, (headers or {}), acceptable_statuses) def get_object_metadata( self, account, container, obj, metadata_prefix='', - acceptable_statuses=(2,)): + acceptable_statuses=(2,), headers=None): """ Gets object metadata. @@ -577,6 +586,7 @@ class InternalClient(object): keys in the dict returned. Defaults to ''. :param acceptable_statuses: List of status for valid responses, defaults to (2,). + :param headers: extra headers to send with request :returns : Dict of object metadata. @@ -587,7 +597,19 @@ class InternalClient(object): """ path = self.make_path(account, container, obj) - return self._get_metadata(path, metadata_prefix, acceptable_statuses) + return self._get_metadata(path, metadata_prefix, acceptable_statuses, + headers=headers) + + def get_object(self, account, container, obj, headers, + acceptable_statuses=(2,)): + """ + Returns a 3-tuple (status, headers, iterator of object body) + """ + + headers = headers or {} + path = self.make_path(account, container, obj) + resp = self.make_request('GET', path, headers, acceptable_statuses) + return (resp.status_int, resp.headers, resp.app_iter) def iter_object_lines( self, account, container, obj, headers=None, diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 3d775fd47a..7aab048310 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -201,6 +201,47 @@ class RestrictedGreenPool(GreenPool): self.waitall() +def pipeline_property(name, **kwargs): + """ + Create a property accessor for the given name. The property will + dig through the bound instance on which it was accessed for an + attribute "app" and check that object for an attribute of the given + name. If the "app" object does not have such an attribute, it will + look for an attribute "app" on THAT object and continue it's search + from there. If the named attribute cannot be found accessing the + property will raise AttributeError. + + If a default kwarg is provided you get that instead of the + AttributeError. When found the attribute will be cached on instance + with the property accessor using the same name as the attribute + prefixed with a leading underscore. + """ + + cache_attr_name = '_%s' % name + + def getter(self): + cached_value = getattr(self, cache_attr_name, None) + if cached_value: + return cached_value + app = self # first app is on self + while True: + app = getattr(app, 'app', None) + if not app: + break + try: + value = getattr(app, name) + except AttributeError: + continue + setattr(self, cache_attr_name, value) + return value + if 'default' in kwargs: + return kwargs['default'] + raise AttributeError('No apps in pipeline have a ' + '%s attribute' % name) + + return property(getter) + + class PipelineWrapper(object): """ This class provides a number of utility methods for @@ -292,6 +333,13 @@ def loadcontext(object_type, uri, name=None, relative_to=None, global_conf=global_conf) +def _add_pipeline_properties(app, *names): + for property_name in names: + if not hasattr(app, property_name): + setattr(app.__class__, property_name, + pipeline_property(property_name)) + + def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True): """ Loads a context from a config file, and if the context is a pipeline diff --git a/swift/obj/server.py b/swift/obj/server.py index a4cc2ac439..2122d81713 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -91,8 +91,9 @@ class ObjectController(object): for header in extra_allowed_headers: if header not in DATAFILE_SYSTEM_META: self.allowed_headers.add(header) - self.expiring_objects_account = \ - (conf.get('auto_create_account_prefix') or '.') + \ + self.auto_create_account_prefix = \ + conf.get('auto_create_account_prefix') or '.' + self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index af8f8737ac..233935254d 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -509,7 +509,8 @@ def get_info(app, env, account, container=None, ret_not_found=False, path = '/v1/%s' % account if container: # Stop and check if we have an account? - if not get_info(app, env, account): + if not get_info(app, env, account) and not account.startswith( + getattr(app, 'auto_create_account_prefix', '.')): return None path += '/' + container diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py index 354b5662ba..346238c419 100644 --- a/swift/proxy/controllers/obj.py +++ b/swift/proxy/controllers/obj.py @@ -196,10 +196,11 @@ class ObjectController(Controller): container_info = self.container_info( self.account_name, self.container_name, req) req.acl = container_info['read_acl'] - policy_idx = container_info['storage_policy'] - obj_ring = self.app.get_object_ring(policy_idx) # pass the policy index to storage nodes via req header - req.headers[POLICY_INDEX] = policy_idx + policy_index = req.headers.get(POLICY_INDEX, + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) + req.headers[POLICY_INDEX] = policy_index if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: @@ -301,10 +302,11 @@ class ObjectController(Controller): self.app.expiring_objects_account, delete_at_container) else: delete_at_container = delete_at_part = delete_at_nodes = None - policy_idx = container_info['storage_policy'] - obj_ring = self.app.get_object_ring(policy_idx) # pass the policy index to storage nodes via req header - req.headers[POLICY_INDEX] = policy_idx + policy_index = req.headers.get(POLICY_INDEX, + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) + req.headers[POLICY_INDEX] = policy_index partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) req.headers['X-Timestamp'] = normalize_timestamp(time.time()) @@ -456,10 +458,11 @@ class ObjectController(Controller): body='If-None-Match only supports *') container_info = self.container_info( self.account_name, self.container_name, req) - policy_idx = container_info['storage_policy'] - obj_ring = self.app.get_object_ring(policy_idx) + policy_index = req.headers.get(POLICY_INDEX, + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) # pass the policy index to storage nodes via req header - req.headers[POLICY_INDEX] = policy_idx + req.headers[POLICY_INDEX] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] @@ -583,6 +586,8 @@ class ObjectController(Controller): source_header = '/%s/%s/%s/%s' % (ver, acct, src_container_name, src_obj_name) source_req = req.copy_get() + # make sure the source request uses it's container_info + source_req.headers.pop(POLICY_INDEX, None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name @@ -771,10 +776,12 @@ class ObjectController(Controller): """HTTP DELETE request handler.""" container_info = self.container_info( self.account_name, self.container_name, req) - policy_idx = container_info['storage_policy'] - obj_ring = self.app.get_object_ring(policy_idx) # pass the policy index to storage nodes via req header - req.headers[POLICY_INDEX] = policy_idx + policy_index = req.headers.get(POLICY_INDEX, + container_info['storage_policy']) + obj_ring = self.app.get_object_ring(policy_index) + # pass the policy index to storage nodes via req header + req.headers[POLICY_INDEX] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] diff --git a/swift/proxy/server.py b/swift/proxy/server.py index ea6207be7f..b62c434fb9 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -112,8 +112,9 @@ class Application(object): [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) - self.expiring_objects_account = \ - (conf.get('auto_create_account_prefix') or '.') + \ + self.auto_create_account_prefix = ( + conf.get('auto_create_account_prefix') or '.') + self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ int(conf.get('expiring_objects_container_divisor') or 86400) diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 52cc624e2f..a9f1d993c6 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -101,5 +101,5 @@ class FakeSwift(object): def call_count(self): return len(self._calls) - def register(self, method, path, response_class, headers, body): + def register(self, method, path, response_class, headers, body=''): self._responses[(method, path)] = (response_class, headers, body) diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index bcc60c4d11..0d969b1814 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -19,10 +19,16 @@ from StringIO import StringIO import unittest from urllib import quote import zlib +from textwrap import dedent +import os from test.unit import FakeLogger from eventlet.green import urllib2 from swift.common import internal_client +from swift.common import swob + +from test.unit import with_tempdir, write_fake_ring, patch_policies +from test.unit.common.middleware.helpers import FakeSwift def not_sleep(seconds): @@ -49,6 +55,21 @@ def make_path(account, container=None, obj=None): return path +def make_path_info(account, container=None, obj=None): + # FakeSwift keys on PATH_INFO - which is *encoded* but unquoted + path = '/v1/%s' % '/'.join( + p for p in (account, container, obj) if p) + return path.encode('utf-8') + + +def get_client_app(): + app = FakeSwift() + with mock.patch('swift.common.internal_client.loadapp', + new=lambda *args, **kwargs: app): + client = internal_client.InternalClient({}, 'test', 1) + return client, app + + class InternalClient(internal_client.InternalClient): def __init__(self): pass @@ -63,7 +84,8 @@ class GetMetadataInternalClient(internal_client.InternalClient): self.get_metadata_called = 0 self.metadata = 'some_metadata' - def _get_metadata(self, path, metadata_prefix, acceptable_statuses=None): + def _get_metadata(self, path, metadata_prefix, acceptable_statuses=None, + headers=None): self.get_metadata_called += 1 self.test.assertEquals(self.path, path) self.test.assertEquals(self.metadata_prefix, metadata_prefix) @@ -179,6 +201,52 @@ class TestCompressingfileReader(unittest.TestCase): class TestInternalClient(unittest.TestCase): + + @patch_policies(legacy_only=True) + @mock.patch('swift.common.utils.HASH_PATH_SUFFIX', new='endcap') + @with_tempdir + def test_load_from_config(self, tempdir): + conf_path = os.path.join(tempdir, 'interal_client.conf') + conf_body = """ + [DEFAULT] + swift_dir = %s + + [pipeline:main] + pipeline = catch_errors cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + auto_create_account_prefix = - + + [filter:cache] + use = egg:swift#memcache + + [filter:catch_errors] + use = egg:swift#catch_errors + """ % tempdir + with open(conf_path, 'w') as f: + f.write(dedent(conf_body)) + account_ring_path = os.path.join(tempdir, 'account.ring.gz') + write_fake_ring(account_ring_path) + container_ring_path = os.path.join(tempdir, 'container.ring.gz') + write_fake_ring(container_ring_path) + object_ring_path = os.path.join(tempdir, 'object.ring.gz') + write_fake_ring(object_ring_path) + client = internal_client.InternalClient(conf_path, 'test', 1) + self.assertEqual(client.account_ring, client.app.app.app.account_ring) + self.assertEqual(client.account_ring.serialized_path, + account_ring_path) + self.assertEqual(client.container_ring, + client.app.app.app.container_ring) + self.assertEqual(client.container_ring.serialized_path, + container_ring_path) + object_ring = client.app.app.app.get_object_ring(0) + self.assertEqual(client.get_object_ring(0), + object_ring) + self.assertEqual(object_ring.serialized_path, + object_ring_path) + self.assertEquals(client.auto_create_account_prefix, '-') + def test_init(self): class App(object): def __init__(self, test, conf_path): @@ -428,21 +496,24 @@ class TestInternalClient(unittest.TestCase): self.assertEquals(1, client.make_request_called) def test_get_metadata_invalid_status(self): - class Response(object): - def __init__(self): - self.status_int = 404 - self.headers = {'some_key': 'some_value'} + class FakeApp(object): + + def __call__(self, environ, start_response): + start_response('404 Not Found', [('x-foo', 'bar')]) + return ['nope'] class InternalClient(internal_client.InternalClient): def __init__(self): - pass - - def make_request(self, *a, **kw): - return Response() + self.user_agent = 'test' + self.request_tries = 1 + self.app = FakeApp() client = InternalClient() - metadata = client._get_metadata('path') - self.assertEquals({}, metadata) + self.assertRaises(internal_client.UnexpectedResponse, + client._get_metadata, 'path') + metadata = client._get_metadata('path', metadata_prefix='x-', + acceptable_statuses=(4,)) + self.assertEqual(metadata, {'foo': 'bar'}) def test_make_path(self): account, container, obj = path_parts() @@ -653,6 +724,26 @@ class TestInternalClient(unittest.TestCase): self.assertEquals(client.metadata, metadata) self.assertEquals(1, client.get_metadata_called) + def test_get_metadadata_with_acceptable_status(self): + account, container, obj = path_parts() + path = make_path_info(account) + client, app = get_client_app() + resp_headers = {'some-important-header': 'some value'} + app.register('GET', path, swob.HTTPOk, resp_headers) + metadata = client.get_account_metadata( + account, acceptable_statuses=(2, 4)) + self.assertEqual(metadata['some-important-header'], + 'some value') + app.register('GET', path, swob.HTTPNotFound, resp_headers) + metadata = client.get_account_metadata( + account, acceptable_statuses=(2, 4)) + self.assertEqual(metadata['some-important-header'], + 'some value') + app.register('GET', path, swob.HTTPServerError, resp_headers) + self.assertRaises(internal_client.UnexpectedResponse, + client.get_account_metadata, account, + acceptable_statuses=(2, 4)) + def test_set_account_metadata(self): account, container, obj = path_parts() path = make_path(account) @@ -823,6 +914,47 @@ class TestInternalClient(unittest.TestCase): self.assertEquals(client.metadata, metadata) self.assertEquals(1, client.get_metadata_called) + def test_get_metadata_extra_headers(self): + class InternalClient(internal_client.InternalClient): + def __init__(self): + self.app = self.fake_app + self.user_agent = 'some_agent' + self.request_tries = 3 + + def fake_app(self, env, start_response): + self.req_env = env + start_response('200 Ok', [('Content-Length', '0')]) + return [] + + client = InternalClient() + headers = {'X-Foo': 'bar'} + client.get_object_metadata('account', 'container', 'obj', + headers=headers) + self.assertEqual(client.req_env['HTTP_X_FOO'], 'bar') + + def test_get_object(self): + account, container, obj = path_parts() + path_info = make_path_info(account, container, obj) + client, app = get_client_app() + headers = {'foo': 'bar'} + body = 'some_object_body' + app.register('GET', path_info, swob.HTTPOk, headers, body) + req_headers = {'x-important-header': 'some_important_value'} + status_int, resp_headers, obj_iter = client.get_object( + account, container, obj, req_headers) + self.assertEqual(status_int // 100, 2) + for k, v in headers.items(): + self.assertEqual(v, resp_headers[k]) + self.assertEqual(''.join(obj_iter), body) + self.assertEqual(resp_headers['content-length'], str(len(body))) + self.assertEqual(app.call_count, 1) + req_headers.update({ + 'host': 'localhost:80', # from swob.Request.blank + 'user-agent': 'test', # from InternalClient.make_request + }) + self.assertEqual(app.calls_with_headers, [( + 'GET', path_info, swob.HeaderKeyDict(req_headers))]) + def test_iter_object_lines(self): class InternalClient(internal_client.InternalClient): def __init__(self, lines): diff --git a/test/unit/common/test_wsgi.py b/test/unit/common/test_wsgi.py index f1573a1f8d..419b94f363 100644 --- a/test/unit/common/test_wsgi.py +++ b/test/unit/common/test_wsgi.py @@ -35,12 +35,15 @@ import swift.common.middleware.catch_errors import swift.common.middleware.gatekeeper import swift.proxy.server +import swift.obj.server as obj_server +import swift.container.server as container_server +import swift.account.server as account_server from swift.common.swob import Request from swift.common import wsgi, utils from swift.common.storage_policy import StoragePolicy, \ StoragePolicyCollection -from test.unit import temptree, write_fake_ring +from test.unit import temptree, with_tempdir, write_fake_ring, patch_policies from paste.deploy import loadwsgi @@ -754,6 +757,7 @@ class TestPipelineWrapper(unittest.TestCase): " catch_errors tempurl proxy-server") +@mock.patch('swift.common.utils.HASH_PATH_SUFFIX', new='endcap') class TestPipelineModification(unittest.TestCase): def pipeline_modules(self, app): # This is rather brittle; it'll break if a middleware stores its app @@ -1008,5 +1012,100 @@ class TestPipelineModification(unittest.TestCase): 'swift.common.middleware.dlo', 'swift.proxy.server']) + @patch_policies + @with_tempdir + def test_loadapp_proxy(self, tempdir): + conf_path = os.path.join(tempdir, 'proxy-server.conf') + conf_body = """ + [DEFAULT] + swift_dir = %s + + [pipeline:main] + pipeline = catch_errors cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + + [filter:catch_errors] + use = egg:swift#catch_errors + """ % tempdir + with open(conf_path, 'w') as f: + f.write(dedent(conf_body)) + account_ring_path = os.path.join(tempdir, 'account.ring.gz') + write_fake_ring(account_ring_path) + container_ring_path = os.path.join(tempdir, 'container.ring.gz') + write_fake_ring(container_ring_path) + object_ring_path = os.path.join(tempdir, 'object.ring.gz') + write_fake_ring(object_ring_path) + object_1_ring_path = os.path.join(tempdir, 'object-1.ring.gz') + write_fake_ring(object_1_ring_path) + app = wsgi.loadapp(conf_path) + proxy_app = app.app.app.app.app + self.assertEqual(proxy_app.account_ring.serialized_path, + account_ring_path) + self.assertEqual(proxy_app.container_ring.serialized_path, + container_ring_path) + self.assertEqual(proxy_app.get_object_ring(0).serialized_path, + object_ring_path) + self.assertEqual(proxy_app.get_object_ring(1).serialized_path, + object_1_ring_path) + + @with_tempdir + def test_loadapp_storage(self, tempdir): + expectations = { + 'object': obj_server.ObjectController, + 'container': container_server.ContainerController, + 'account': account_server.AccountController, + } + + for server_type, controller in expectations.items(): + conf_path = os.path.join( + tempdir, '%s-server.conf' % server_type) + conf_body = """ + [DEFAULT] + swift_dir = %s + + [app:main] + use = egg:swift#%s + """ % (tempdir, server_type) + with open(conf_path, 'w') as f: + f.write(dedent(conf_body)) + app = wsgi.loadapp(conf_path) + self.assertTrue(isinstance(app, controller)) + + def test_pipeline_property(self): + depth = 3 + + class FakeApp(object): + pass + + class AppFilter(object): + + def __init__(self, app): + self.app = app + + # make a pipeline + app = FakeApp() + filtered_app = app + for i in range(depth): + filtered_app = AppFilter(filtered_app) + + # AttributeError if no apps in the pipeline have attribute + wsgi._add_pipeline_properties(filtered_app, 'foo') + self.assertRaises(AttributeError, getattr, filtered_app, 'foo') + + # set the attribute + self.assert_(isinstance(app, FakeApp)) + app.foo = 'bar' + self.assertEqual(filtered_app.foo, 'bar') + + # attribute is cached + app.foo = 'baz' + self.assertEqual(filtered_app.foo, 'bar') + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/controllers/test_base.py b/test/unit/proxy/controllers/test_base.py index cf85ffe6e4..2c150e1c08 100644 --- a/test/unit/proxy/controllers/test_base.py +++ b/test/unit/proxy/controllers/test_base.py @@ -13,15 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools +from collections import defaultdict import unittest from mock import patch from swift.proxy.controllers.base import headers_to_container_info, \ headers_to_account_info, headers_to_object_info, get_container_info, \ get_container_memcache_key, get_account_info, get_account_memcache_key, \ - get_object_env_key, _get_cache_key, get_info, get_object_info, \ - Controller, GetOrHeadHandler -from swift.common.swob import Request, HTTPException, HeaderKeyDict + get_object_env_key, get_info, get_object_info, \ + Controller, GetOrHeadHandler, _set_info_cache, _set_object_info_cache +from swift.common.swob import Request, HTTPException, HeaderKeyDict, \ + RESPONSE_REASONS from swift.common.utils import split_path +from swift.common.http import is_success from swift.common.storage_policy import StoragePolicy from test.unit import fake_http_connect, FakeRing, FakeMemcache from swift.proxy import server as proxy_server @@ -30,58 +34,119 @@ from swift.common.request_helpers import get_sys_meta_prefix from test.unit import patch_policies -FakeResponse_status_int = 201 - - class FakeResponse(object): - def __init__(self, headers, env, account, container, obj): - self.headers = headers - self.status_int = FakeResponse_status_int - self.environ = env + + base_headers = {} + + def __init__(self, status_int=200, headers=None, body=''): + self.status_int = status_int + self._headers = headers or {} + self.body = body + + @property + def headers(self): + if is_success(self.status_int): + self._headers.update(self.base_headers) + return self._headers + + +class AccountResponse(FakeResponse): + + base_headers = { + 'x-account-container-count': 333, + 'x-account-object-count': 1000, + 'x-account-bytes-used': 6666, + } + + +class ContainerResponse(FakeResponse): + + base_headers = { + 'x-container-object-count': 1000, + 'x-container-bytes-used': 6666, + } + + +class ObjectResponse(FakeResponse): + + base_headers = { + 'content-length': 5555, + 'content-type': 'text/plain' + } + + +class DynamicResponseFactory(object): + + def __init__(self, *statuses): + if statuses: + self.statuses = iter(statuses) + else: + self.statuses = itertools.repeat(200) + self.stats = defaultdict(int) + + response_type = { + 'obj': ObjectResponse, + 'container': ContainerResponse, + 'account': AccountResponse, + } + + def _get_response(self, type_): + self.stats[type_] += 1 + class_ = self.response_type[type_] + return class_(self.statuses.next()) + + def get_response(self, environ): + (version, account, container, obj) = split_path( + environ['PATH_INFO'], 2, 4, True) if obj: - env_key = get_object_env_key(account, container, obj) + resp = self._get_response('obj') + elif container: + resp = self._get_response('container') else: - cache_key, env_key = _get_cache_key(account, container) - - if account and container and obj: - info = headers_to_object_info(headers, FakeResponse_status_int) - elif account and container: - info = headers_to_container_info(headers, FakeResponse_status_int) - else: - info = headers_to_account_info(headers, FakeResponse_status_int) - env[env_key] = info + resp = self._get_response('account') + resp.account = account + resp.container = container + resp.obj = obj + return resp -class FakeRequest(object): - def __init__(self, env, path, swift_source=None): - self.environ = env - (version, account, container, obj) = split_path(path, 2, 4, True) - self.account = account - self.container = container - self.obj = obj - if obj: - stype = 'object' - self.headers = {'content-length': 5555, - 'content-type': 'text/plain'} - else: - stype = container and 'container' or 'account' - self.headers = {'x-%s-object-count' % (stype): 1000, - 'x-%s-bytes-used' % (stype): 6666} - if swift_source: - meta = 'x-%s-meta-fakerequest-swift-source' % stype - self.headers[meta] = swift_source +class FakeApp(object): - def get_response(self, app): - return FakeResponse(self.headers, self.environ, self.account, - self.container, self.obj) + recheck_container_existence = 30 + recheck_account_existence = 30 + + def __init__(self, response_factory=None, statuses=None): + self.responses = response_factory or \ + DynamicResponseFactory(*statuses or []) + self.sources = [] + + def __call__(self, environ, start_response): + self.sources.append(environ.get('swift.source')) + response = self.responses.get_response(environ) + reason = RESPONSE_REASONS[response.status_int][0] + start_response('%d %s' % (response.status_int, reason), + [(k, v) for k, v in response.headers.items()]) + # It's a bit strnage, but the get_info cache stuff relies on the + # app setting some keys in the environment as it makes requests + # (in particular GETorHEAD_base) - so our fake does the same + _set_info_cache(self, environ, response.account, + response.container, response) + if response.obj: + _set_object_info_cache(self, environ, response.account, + response.container, response.obj, + response) + return iter(response.body) -class FakeCache(object): - def __init__(self, val): - self.val = val +class FakeCache(FakeMemcache): + def __init__(self, stub=None, **pre_cached): + super(FakeCache, self).__init__() + if pre_cached: + self.store.update(pre_cached) + self.stub = stub - def get(self, *args): - return self.val + def get(self, key): + return self.stub or self.store.get(key) @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) @@ -125,144 +190,158 @@ class TestFuncs(unittest.TestCase): self.assertEqual(resp.environ['swift.account/a']['status'], 200) def test_get_info(self): - global FakeResponse_status_int + app = FakeApp() # Do a non cached call to account env = {} - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - info_a = get_info(None, env, 'a') + info_a = get_info(app, env, 'a') # Check that you got proper info - self.assertEquals(info_a['status'], 201) + self.assertEquals(info_a['status'], 200) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) + # Make sure the app was called + self.assertEqual(app.responses.stats['account'], 1) # Do an env cached call to account - info_a = get_info(None, env, 'a') + info_a = get_info(app, env, 'a') # Check that you got proper info - self.assertEquals(info_a['status'], 201) + self.assertEquals(info_a['status'], 200) self.assertEquals(info_a['bytes'], 6666) self.assertEquals(info_a['total_object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) + # Make sure the app was NOT called AGAIN + self.assertEqual(app.responses.stats['account'], 1) # This time do env cached call to account and non cached to container - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - info_c = get_info(None, env, 'a', 'c') + info_c = get_info(app, env, 'a', 'c') # Check that you got proper info - self.assertEquals(info_a['status'], 201) + self.assertEquals(info_c['status'], 200) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) self.assertEquals(env.get('swift.container/a/c'), info_c) + # Make sure the app was called for container + self.assertEqual(app.responses.stats['container'], 1) # This time do a non cached call to account than non cached to # container + app = FakeApp() env = {} # abandon previous call to env - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - info_c = get_info(None, env, 'a', 'c') + info_c = get_info(app, env, 'a', 'c') # Check that you got proper info - self.assertEquals(info_a['status'], 201) + self.assertEquals(info_c['status'], 200) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) self.assertEquals(env.get('swift.container/a/c'), info_c) + # check app calls both account and container + self.assertEqual(app.responses.stats['account'], 1) + self.assertEqual(app.responses.stats['container'], 1) # This time do an env cached call to container while account is not # cached del(env['swift.account/a']) - info_c = get_info(None, env, 'a', 'c') + info_c = get_info(app, env, 'a', 'c') # Check that you got proper info - self.assertEquals(info_a['status'], 201) + self.assertEquals(info_a['status'], 200) self.assertEquals(info_c['bytes'], 6666) self.assertEquals(info_c['object_count'], 1000) # Make sure the env cache is set and account still not cached self.assertEquals(env.get('swift.container/a/c'), info_c) + # no additional calls were made + self.assertEqual(app.responses.stats['account'], 1) + self.assertEqual(app.responses.stats['container'], 1) # Do a non cached call to account not found with ret_not_found + app = FakeApp(statuses=(404,)) env = {} - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - try: - FakeResponse_status_int = 404 - info_a = get_info(None, env, 'a', ret_not_found=True) - finally: - FakeResponse_status_int = 201 + info_a = get_info(app, env, 'a', ret_not_found=True) # Check that you got proper info self.assertEquals(info_a['status'], 404) - self.assertEquals(info_a['bytes'], 6666) - self.assertEquals(info_a['total_object_count'], 1000) + self.assertEquals(info_a['bytes'], None) + self.assertEquals(info_a['total_object_count'], None) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) + # and account was called + self.assertEqual(app.responses.stats['account'], 1) # Do a cached call to account not found with ret_not_found - info_a = get_info(None, env, 'a', ret_not_found=True) + info_a = get_info(app, env, 'a', ret_not_found=True) # Check that you got proper info self.assertEquals(info_a['status'], 404) - self.assertEquals(info_a['bytes'], 6666) - self.assertEquals(info_a['total_object_count'], 1000) + self.assertEquals(info_a['bytes'], None) + self.assertEquals(info_a['total_object_count'], None) # Make sure the env cache is set self.assertEquals(env.get('swift.account/a'), info_a) + # add account was NOT called AGAIN + self.assertEqual(app.responses.stats['account'], 1) # Do a non cached call to account not found without ret_not_found + app = FakeApp(statuses=(404,)) env = {} - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - try: - FakeResponse_status_int = 404 - info_a = get_info(None, env, 'a') - finally: - FakeResponse_status_int = 201 + info_a = get_info(app, env, 'a') # Check that you got proper info self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) + # and account was called + self.assertEqual(app.responses.stats['account'], 1) # Do a cached call to account not found without ret_not_found info_a = get_info(None, env, 'a') # Check that you got proper info self.assertEquals(info_a, None) self.assertEquals(env['swift.account/a']['status'], 404) + # add account was NOT called AGAIN + self.assertEqual(app.responses.stats['account'], 1) def test_get_container_info_swift_source(self): - req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_container_info(req.environ, 'app', swift_source='MC') - self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + app = FakeApp() + req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache()}) + get_container_info(req.environ, app, swift_source='MC') + self.assertEqual(app.sources, ['GET_INFO', 'MC']) def test_get_object_info_swift_source(self): + app = FakeApp() req = Request.blank("/v1/a/c/o", - environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_object_info(req.environ, 'app', swift_source='LU') - self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU') + environ={'swift.cache': FakeCache()}) + get_object_info(req.environ, app, swift_source='LU') + self.assertEqual(app.sources, ['LU']) def test_get_container_info_no_cache(self): req = Request.blank("/v1/AUTH_account/cont", environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_container_info(req.environ, 'xxx') + resp = get_container_info(req.environ, FakeApp()) self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['object_count'], 1000) + def test_get_container_info_no_account(self): + responses = DynamicResponseFactory(404, 200) + app = FakeApp(responses) + req = Request.blank("/v1/AUTH_does_not_exist/cont") + info = get_container_info(req.environ, app) + self.assertEqual(info['status'], 0) + + def test_get_container_info_no_auto_account(self): + responses = DynamicResponseFactory(404, 200) + app = FakeApp(responses) + req = Request.blank("/v1/.system_account/cont") + info = get_container_info(req.environ, app) + self.assertEqual(info['status'], 200) + self.assertEquals(info['bytes'], 6666) + self.assertEquals(info['object_count'], 1000) + def test_get_container_info_cache(self): - cached = {'status': 404, - 'bytes': 3333, - 'object_count': 10, - # simplejson sometimes hands back strings, sometimes unicodes - 'versions': u"\u1F4A9"} + cache_stub = { + 'status': 404, 'bytes': 3333, 'object_count': 10, + # simplejson sometimes hands back strings, sometimes unicodes + 'versions': u"\u1F4A9"} req = Request.blank("/v1/account/cont", - environ={'swift.cache': FakeCache(cached)}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_container_info(req.environ, 'xxx') + environ={'swift.cache': FakeCache(cache_stub)}) + resp = get_container_info(req.environ, FakeApp()) self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['object_count'], 10) self.assertEquals(resp['status'], 404) @@ -278,18 +357,16 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['bytes'], 3867) def test_get_account_info_swift_source(self): - req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_account_info(req.environ, 'a', swift_source='MC') - self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC') + app = FakeApp() + req = Request.blank("/v1/a", environ={'swift.cache': FakeCache()}) + get_account_info(req.environ, app, swift_source='MC') + self.assertEqual(app.sources, ['MC']) def test_get_account_info_no_cache(self): + app = FakeApp() req = Request.blank("/v1/AUTH_account", environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_account_info(req.environ, 'xxx') + resp = get_account_info(req.environ, app) self.assertEquals(resp['bytes'], 6666) self.assertEquals(resp['total_object_count'], 1000) @@ -300,9 +377,7 @@ class TestFuncs(unittest.TestCase): 'total_object_count': 10} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_account_info(req.environ, 'xxx') + resp = get_account_info(req.environ, FakeApp()) self.assertEquals(resp['bytes'], 3333) self.assertEquals(resp['total_object_count'], 10) self.assertEquals(resp['status'], 404) @@ -315,9 +390,7 @@ class TestFuncs(unittest.TestCase): 'meta': {}} req = Request.blank("/v1/account/cont", environ={'swift.cache': FakeCache(cached)}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_account_info(req.environ, 'xxx') + resp = get_account_info(req.environ, FakeApp()) self.assertEquals(resp['status'], 404) self.assertEquals(resp['bytes'], '3333') self.assertEquals(resp['container_count'], 234) @@ -347,11 +420,13 @@ class TestFuncs(unittest.TestCase): self.assertEquals(resp['type'], 'application/json') def test_get_object_info_no_env(self): + app = FakeApp() req = Request.blank("/v1/account/cont/obj", environ={'swift.cache': FakeCache({})}) - with patch('swift.proxy.controllers.base.' - '_prepare_pre_auth_info_request', FakeRequest): - resp = get_object_info(req.environ, 'xxx') + resp = get_object_info(req.environ, app) + self.assertEqual(app.responses.stats['account'], 0) + self.assertEqual(app.responses.stats['container'], 0) + self.assertEqual(app.responses.stats['obj'], 1) self.assertEquals(resp['length'], 5555) self.assertEquals(resp['type'], 'text/plain') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index d57ad2851f..bd554e7c9e 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -19,6 +19,7 @@ import sys import unittest from contextlib import contextmanager, nested from shutil import rmtree +from StringIO import StringIO import gc import time from textwrap import dedent @@ -46,7 +47,7 @@ from swift.container import server as container_server from swift.obj import server as object_server from swift.common.middleware import proxy_logging from swift.common.middleware.acl import parse_acl, format_acl -from swift.common.exceptions import ChunkReadTimeout +from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist from swift.common import utils, constraints from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.wsgi import monkey_patch_mimetools, loadapp @@ -1022,6 +1023,80 @@ class TestObjectController(unittest.TestCase): check_file(2, 'c2', ['sde1', 'sdf1'], True) check_file(2, 'c2', ['sda1', 'sdb1', 'sdc1', 'sdd1'], False) + @unpatch_policies + def test_policy_IO_override(self): + if hasattr(_test_servers[-1], '_filesystem'): + # ironically, the _filesystem attribute on the object server means + # the in-memory diskfile is in use, so this test does not apply + return + + prosrv = _test_servers[0] + + # validate container policy is 1 + req = Request.blank('/v1/a/c1', method='HEAD') + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 204) # sanity check + self.assertEqual(POLICIES[1].name, res.headers['x-storage-policy']) + + # check overrides: put it in policy 2 (not where the container says) + req = Request.blank( + '/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'PUT', + 'wsgi.input': StringIO("hello")}, + headers={'Content-Type': 'text/plain', + 'Content-Length': '5', + 'X-Backend-Storage-Policy-Index': '2'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 201) # sanity check + + # go to disk to make sure it's there + partition, nodes = prosrv.get_object_ring(2).get_nodes( + 'a', 'c1', 'wrong-o') + node = nodes[0] + conf = {'devices': _testdir, 'mount_check': 'false'} + df_mgr = diskfile.DiskFileManager(conf, FakeLogger()) + df = df_mgr.get_diskfile(node['device'], partition, 'a', + 'c1', 'wrong-o', policy_idx=2) + with df.open(): + contents = ''.join(df.reader()) + self.assertEqual(contents, "hello") + + # can't get it from the normal place + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': 'text/plain'}) + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 404) # sanity check + + # but we can get it from policy 2 + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Content-Type': 'text/plain', + 'X-Backend-Storage-Policy-Index': '2'}) + + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 200) + self.assertEqual(res.body, 'hello') + + # and we can delete it the same way + req = Request.blank('/v1/a/c1/wrong-o', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Content-Type': 'text/plain', + 'X-Backend-Storage-Policy-Index': '2'}) + + res = req.get_response(prosrv) + self.assertEqual(res.status_int, 204) + + df = df_mgr.get_diskfile(node['device'], partition, 'a', + 'c1', 'wrong-o', policy_idx=2) + try: + df.open() + except DiskFileNotExist as e: + now = time.time() + self.assert_(now - 1 < float(e.timestamp) < now + 1) + else: + self.fail('did not raise DiskFileNotExist') + @unpatch_policies def test_GET_newest_large_file(self): prolis = _test_sockets[0] @@ -1599,6 +1674,125 @@ class TestObjectController(unittest.TestCase): test_status_map((200, 200, 404, 500, 500), 503) test_status_map((200, 200, 404, 404, 404), 404) + @patch_policies([ + StoragePolicy(0, 'zero', is_default=True, object_ring=FakeRing()), + StoragePolicy(1, 'one', object_ring=FakeRing()), + ]) + def test_POST_backend_headers(self): + self.app.object_post_as_copy = False + self.app.sort_nodes = lambda nodes: nodes + backend_requests = [] + + def capture_requests(ip, port, method, path, headers, *args, + **kwargs): + backend_requests.append((method, path, headers)) + + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue'}) + + # we want the container_info response to says a policy index of 1 + resp_headers = {'X-Backend-Storage-Policy-Index': 1} + with mocked_http_conn( + 200, 200, 202, 202, 202, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 5) + + def check_request(req, method, path, headers=None): + req_method, req_path, req_headers = req + self.assertEqual(method, req_method) + # caller can ignore leading path parts + self.assertTrue(req_path.endswith(path)) + headers = headers or {} + # caller can ignore some headers + for k, v in headers.items(): + self.assertEqual(req_headers[k], v) + account_request = backend_requests.pop(0) + check_request(account_request, method='HEAD', path='/sda/1/a') + container_request = backend_requests.pop(0) + check_request(container_request, method='HEAD', path='/sda/1/a/c') + for i, (device, request) in enumerate(zip(('sda', 'sdb', 'sdc'), + backend_requests)): + expectations = { + 'method': 'POST', + 'path': '/%s/1/a/c/o' % device, + 'headers': { + 'X-Container-Host': '10.0.0.%d:100%d' % (i, i), + 'X-Container-Partition': '1', + 'Connection': 'close', + 'User-Agent': 'proxy-server %s' % os.getpid(), + 'Host': 'localhost:80', + 'X-Container-Device': device, + 'Referer': 'POST http://localhost/v1/a/c/o', + 'X-Object-Meta-Color': 'Blue', + POLICY_INDEX: '1' + }, + } + check_request(request, **expectations) + + # and again with policy override + self.app.memcache.store = {} + backend_requests = [] + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue', + POLICY_INDEX: 0}) + with mocked_http_conn( + 200, 200, 202, 202, 202, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 5) + for request in backend_requests[2:]: + expectations = { + 'method': 'POST', + 'path': '/1/a/c/o', # ignore device bit + 'headers': { + 'X-Object-Meta-Color': 'Blue', + POLICY_INDEX: '0', + } + } + check_request(request, **expectations) + + # and this time with post as copy + self.app.object_post_as_copy = True + self.app.memcache.store = {} + backend_requests = [] + req = Request.blank('/v1/a/c/o', {}, method='POST', + headers={'X-Object-Meta-Color': 'Blue', + POLICY_INDEX: 0}) + with mocked_http_conn( + 200, 200, 200, 200, 200, 201, 201, 201, + headers=resp_headers, give_connect=capture_requests + ) as fake_conn: + resp = req.get_response(self.app) + self.assertRaises(StopIteration, fake_conn.code_iter.next) + self.assertEqual(resp.status_int, 202) + self.assertEqual(len(backend_requests), 8) + policy0 = {POLICY_INDEX: '0'} + policy1 = {POLICY_INDEX: '1'} + expected = [ + # account info + {'method': 'HEAD', 'path': '/1/a'}, + # container info + {'method': 'HEAD', 'path': '/1/a/c'}, + # x-newests + {'method': 'GET', 'path': '/1/a/c/o', 'headers': policy1}, + {'method': 'GET', 'path': '/1/a/c/o', 'headers': policy1}, + {'method': 'GET', 'path': '/1/a/c/o', 'headers': policy1}, + # new writes + {'method': 'PUT', 'path': '/1/a/c/o', 'headers': policy0}, + {'method': 'PUT', 'path': '/1/a/c/o', 'headers': policy0}, + {'method': 'PUT', 'path': '/1/a/c/o', 'headers': policy0}, + ] + for request, expectations in zip(backend_requests, expected): + check_request(request, **expectations) + def test_POST_as_copy(self): with save_globals(): def test_status_map(statuses, expected):