From 99be1615e0c4f3af7498dc7822f9ccc96dbaf3dc Mon Sep 17 00:00:00 2001 From: Dan Hersam Date: Fri, 10 May 2013 20:50:56 +0000 Subject: [PATCH] Treat directory objects as not found Make StaticWeb search for an index file rather than returning a 0-byte object. Add new StaticWeb header to identify a directory marker object: X-Container-Meta-Web-Directory-Type (default to application/directory) so user can specify the content type for directory marker objects. This required requesting the container headers earlier in the code and clearing the value of _error for non-404 errors. Add unit tests for directory object scenarios. Fix end tags in an unrelated HTML block. Bug 1178817 DocImpact Change-Id: I561f00c099feaa82fd658f5050bd90c76717da24 --- swift/common/middleware/staticweb.py | 105 ++++++++++++++---- test/unit/common/middleware/test_staticweb.py | 83 ++++++++++++-- 2 files changed, 155 insertions(+), 33 deletions(-) diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index a73144cbfd..adc36c52f4 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -67,6 +67,12 @@ the .../listing.css style sheet. If you "view source" in your browser on a listing page, you will see the well defined document structure that can be styled. +The content-type of directory marker objects can be modified by setting +the ``X-Container-Meta-Web-Directory-Type`` header. If the header is not set, +application/directory is used by default. Directory marker objects are +0-byte objects that represent directories to create a simulated hierarchical +structure. + Example usage of this middleware via ``swift``: Make the container publicly readable:: @@ -99,6 +105,13 @@ Example usage of this middleware via ``swift``: swift post -m 'web-error:error.html' container Now 401's should load 401error.html, 404's should load 404error.html, etc. + + Set Content-Type of directory marker object:: + + swift post -m 'web-directory-type:text/directory' container + + Now 0-byte objects with a content-type of text/directory will be treated + as directories rather than objects. """ @@ -124,6 +137,23 @@ def quote(value, safe='/'): return urllib_quote(value, safe) +def get_memcache_key(version, account, container): + """ + This key's value is (index, error, listings, listings_css, dir_type) + """ + return '/staticweb2/%s/%s/%s' % (version, account, container) + + +def get_compat_memcache_key(version, account, container): + """ + This key's value is (index, error, listings, listings_css) + + TODO: This compat key and its use should be removed after the + Havana OpenStack release. + """ + return '/staticweb/%s/%s/%s' % (version, account, container) + + class _StaticWebContext(WSGIContext): """ The Static Web WSGI middleware filter; serves container data as a @@ -145,7 +175,8 @@ class _StaticWebContext(WSGIContext): self.cache_timeout = staticweb.cache_timeout self.agent = '%(orig)s StaticWeb' # Results from the last call to self._get_container_info. - self._index = self._error = self._listings = self._listings_css = None + self._index = self._error = self._listings = self._listings_css = \ + self._dir_type = None def _error_response(self, response, env, start_response): """ @@ -179,22 +210,32 @@ class _StaticWebContext(WSGIContext): def _get_container_info(self, env): """ Retrieves x-container-meta-web-index, x-container-meta-web-error, - x-container-meta-web-listings, and x-container-meta-web-listings-css - from memcache or from the cluster and stores the result in memcache and - in self._index, self._error, self._listings, and self._listings_css. + x-container-meta-web-listings, x-container-meta-web-listings-css, + and x-container-meta-web-directory-type from memcache or from the + cluster and stores the result in memcache and in self._index, + self._error, self._listings, self._listings_css and self._dir_type. :param env: The WSGI environment dict. """ - self._index = self._error = self._listings = self._listings_css = None + self._index = self._error = self._listings = self._listings_css = \ + self._dir_type = None memcache_client = cache_from_env(env) if memcache_client: - memcache_key = '/staticweb/%s/%s/%s' % (self.version, self.account, - self.container) - cached_data = memcache_client.get(memcache_key) + cached_data = memcache_client.get( + get_memcache_key(self.version, self.account, self.container)) if cached_data: - (self._index, self._error, self._listings, - self._listings_css) = cached_data + (self._index, self._error, self._listings, self._listings_css, + self._dir_type) = cached_data return + else: + cached_data = memcache_client.get( + get_compat_memcache_key( + self.version, self.account, self.container)) + if cached_data: + (self._index, self._error, self._listings, + self._listings_css) = cached_data + self._dir_type = '' + return resp = make_pre_authed_request( env, 'HEAD', '/%s/%s/%s' % ( self.version, self.account, self.container), @@ -209,11 +250,16 @@ class _StaticWebContext(WSGIContext): self._listings_css = \ resp.headers.get('x-container-meta-web-listings-css', '').strip() + self._dir_type = \ + resp.headers.get('x-container-meta-web-directory-type', + '').strip() if memcache_client: - memcache_client.set(memcache_key, - (self._index, self._error, self._listings, - self._listings_css), - time=self.cache_timeout) + memcache_client.set( + get_memcache_key( + self.version, self.account, self.container), + (self._index, self._error, self._listings, + self._listings_css, self._dir_type), + time=self.cache_timeout) def _listing(self, env, start_response, prefix=None): """ @@ -378,13 +424,25 @@ class _StaticWebContext(WSGIContext): tmp_env['swift.source'] = 'SW' resp = self._app_call(tmp_env) status_int = self._get_status_int() - if is_success(status_int) or is_redirection(status_int): - start_response(self._response_status, self._response_headers, - self._response_exc_info) - return resp - if status_int != HTTP_NOT_FOUND: - return self._error_response(resp, env, start_response) self._get_container_info(env) + if is_success(status_int) or is_redirection(status_int): + # Treat directory marker objects as not found + if not self._dir_type: + self._dir_type = 'application/directory' + content_length = self._response_header_value('content-length') + content_length = int(content_length) if content_length else 0 + if self._response_header_value('content-type') == self._dir_type \ + and content_length <= 1: + status_int = HTTP_NOT_FOUND + else: + start_response(self._response_status, self._response_headers, + self._response_exc_info) + return resp + if status_int != HTTP_NOT_FOUND: + # Retaining the previous code's behavior of not using custom error + # pages for non-404 errors. + self._error = None + return self._error_response(resp, env, start_response) if not self._listings and not self._index: return self.app(env, start_response) status_int = HTTP_NOT_FOUND @@ -461,9 +519,10 @@ class StaticWeb(object): if env['REQUEST_METHOD'] in ('PUT', 'POST') and container and not obj: memcache_client = cache_from_env(env) if memcache_client: - memcache_key = \ - '/staticweb/%s/%s/%s' % (version, account, container) - memcache_client.delete(memcache_key) + memcache_client.delete( + get_memcache_key(version, account, container)) + memcache_client.delete( + get_compat_memcache_key(version, account, container)) return self.app(env, start_response) if env['REQUEST_METHOD'] not in ('HEAD', 'GET'): return self.app(env, start_response) diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py index b0eff3d9bf..1f4f9ba75b 100644 --- a/test/unit/common/middleware/test_staticweb.py +++ b/test/unit/common/middleware/test_staticweb.py @@ -137,7 +137,8 @@ class FakeApp(object): 'x-container-meta-web-index': 'index.html', 'x-container-meta-web-error': 'error.html', 'x-container-meta-web-listings': 't', - 'x-container-meta-web-listings-css': 'listing.css'}) + 'x-container-meta-web-listings-css': 'listing.css', + 'x-container-meta-web-directory-type': 'text/dir'}) elif env['PATH_INFO'] == '/v1/a/c4/one.txt': return Response(status='200 Ok', headers={'x-object-meta-test': 'value'}, @@ -160,8 +161,8 @@ class FakeApp(object):

Chrome's 404 fancy-page sucks.

- - + + '''.strip())(env, start_response) elif env['PATH_INFO'] == '/v1/a/c5': return self.listing(env, start_response, @@ -216,6 +217,47 @@ class FakeApp(object): return Response(status='404 Not Found')(env, start_response) elif env['PATH_INFO'] == '/v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/': return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] in ('/v1/a/c11', '/v1/a/c11/'): + return self.listing(env, start_response, + {'x-container-read': '.r:*', + 'x-container-meta-web-index': 'index.html'}) + elif env['PATH_INFO'] == '/v1/a/c11/subdir/': + return Response(status='200 Ok', headers={'Content-Type':\ + 'application/directory'})(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11/subdir/index.html': + return Response(status='200 Ok', body=''' + + +

c11 subdir index

+ + + '''.strip())(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11/subdir2/': + return Response(status='200 Ok', headers={'Content-Type':\ + 'application/directory'})(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11/subdir2/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] in ('/v1/a/c11a', '/v1/a/c11a/'): + return self.listing(env, start_response, + {'x-container-read': '.r:*', + 'x-container-meta-web-index': 'index.html', + 'x-container-meta-web-directory-type': \ + 'text/directory'}) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir/': + return Response(status='200 Ok', headers={'Content-Type':\ + 'text/directory'})(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir2/': + return Response(status='200 Ok', headers={'Content-Type':\ + 'application/directory'})(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir2/index.html': + return Response(status='404 Not Found')(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir3/': + return Response(status='200 Ok', headers={'Content-Type':\ + 'not_a/directory'})(env, start_response) + elif env['PATH_INFO'] == '/v1/a/c11a/subdir3/index.html': + return Response(status='404 Not Found')(env, start_response) else: raise Exception('Unknown path %r' % env['PATH_INFO']) @@ -535,8 +577,8 @@ class TestStaticWeb(unittest.TestCase): ).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 301) self.assertEquals(fake_memcache.store, - {'/staticweb/v1/a/c4': - ('index.html', 'error.html', 't', 'listing.css')}) + {'/staticweb2/v1/a/c4': + ('index.html', 'error.html', 't', 'listing.css', 'text/dir')}) self.assert_(self.test_staticweb.app.get_c4_called) self.test_staticweb.app.get_c4_called = False resp = Request.blank('/v1/a/c4', @@ -545,8 +587,8 @@ class TestStaticWeb(unittest.TestCase): self.assertEquals(resp.status_int, 301) self.assert_(not self.test_staticweb.app.get_c4_called) self.assertEquals(fake_memcache.store, - {'/staticweb/v1/a/c4': - ('index.html', 'error.html', 't', 'listing.css')}) + {'/staticweb2/v1/a/c4': + ('index.html', 'error.html', 't', 'listing.css', 'text/dir')}) resp = Request.blank('/v1/a/c4', environ={'swift.cache': fake_memcache, 'REQUEST_METHOD': 'PUT'} ).get_response(self.test_staticweb) @@ -557,8 +599,8 @@ class TestStaticWeb(unittest.TestCase): ).get_response(self.test_staticweb) self.assertEquals(resp.status_int, 301) self.assertEquals(fake_memcache.store, - {'/staticweb/v1/a/c4': - ('index.html', 'error.html', 't', 'listing.css')}) + {'/staticweb2/v1/a/c4': + ('index.html', 'error.html', 't', 'listing.css', 'text/dir')}) resp = Request.blank('/v1/a/c4', environ={'swift.cache': fake_memcache, 'REQUEST_METHOD': 'POST'} ).get_response(self.test_staticweb) @@ -655,13 +697,34 @@ class TestStaticWeb(unittest.TestCase): self.assert_( 'Listing of /v1/a/c10/\xe2\x98\x83/\xe2\x98\x83/' in resp.body) + def test_container11subdirmarkerobjectindex(self): + resp = Request.blank('/v1/a/c11/subdir/').get_response( + self.test_staticweb) + self.assertEquals(resp.status_int, 200) + self.assert_('

c11 subdir index

' in resp.body) + + def test_container11subdirmarkermatchdirtype(self): + resp = Request.blank('/v1/a/c11a/subdir/').get_response( + self.test_staticweb) + self.assertEquals(resp.status_int, 404) + + def test_container11subdirmarkeraltdirtype(self): + resp = Request.blank('/v1/a/c11a/subdir2/').get_response( + self.test_staticweb) + self.assertEquals(resp.status_int, 200) + + def test_container11subdirmarkerinvaliddirtype(self): + resp = Request.blank('/v1/a/c11a/subdir3/').get_response( + self.test_staticweb) + self.assertEquals(resp.status_int, 200) + def test_subrequest_once_if_possible(self): resp = Request.blank( '/v1/a/c4/one.txt').get_response(self.test_staticweb) self.assertEquals(resp.status_int, 200) self.assertEquals(resp.headers['x-object-meta-test'], 'value') self.assertEquals(resp.body, '1') - self.assertEquals(self.app.calls, 1) + self.assertEquals(self.app.calls, 2) if __name__ == '__main__':