From 8b11ce521f68909da3464987a7c8be37644c27a0 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Thu, 25 Jul 2013 15:43:18 -0700 Subject: [PATCH] return the SLO etag generated from the segment etags on PUT Change-Id: Idf742035da51170ea6a4d6278422ef307a46e121 --- swift/common/middleware/slo.py | 78 +++++++++++++++++-------- test/unit/common/middleware/test_slo.py | 33 +++++++---- 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index b0f0131680..34ff158d26 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -138,12 +138,14 @@ from urllib import quote from cStringIO import StringIO from datetime import datetime import mimetypes +from hashlib import md5 from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ HTTPMethodNotAllowed, HTTPRequestEntityTooLarge, HTTPLengthRequired, \ - HTTPOk, HTTPPreconditionFailed, wsgify + HTTPOk, HTTPPreconditionFailed, HTTPException from swift.common.utils import json, get_logger, config_true_value from swift.common.constraints import check_utf8, MAX_BUFFERED_SLO_SEGMENTS from swift.common.http import HTTP_NOT_FOUND +from swift.common.wsgi import WSGIContext from swift.common.middleware.bulk import get_response_body, \ ACCEPTABLE_FORMATS, Bulk @@ -171,6 +173,26 @@ def parse_input(raw_data): return parsed_data +class SloContext(WSGIContext): + + def __init__(self, slo, slo_etag): + WSGIContext.__init__(self, slo.app) + self.slo_etag = '"' + slo_etag.hexdigest() + '"' + + def handle_slo_put(self, req, start_response): + app_resp = self._app_call(req.environ) + + for i in xrange(len(self._response_headers)): + if self._response_headers[i][0].lower() == 'etag': + self._response_headers[i] = ('Etag', self.slo_etag) + break + + start_response(self._response_status, + self._response_headers, + self._response_exc_info) + return app_resp + + class StaticLargeObject(object): """ StaticLargeObject Middleware @@ -197,11 +219,12 @@ class StaticLargeObject(object): self.bulk_deleter = Bulk( app, {'max_deletes_per_request': self.max_manifest_segments}) - def handle_multipart_put(self, req): + def handle_multipart_put(self, req, start_response): """ Will handle the PUT of a SLO manifest. Heads every object in manifest to check if is valid and if so will - save a manifest generated from the user input. + save a manifest generated from the user input. Uses WSGIContext to + call self.app and start_response and returns a WSGI iterator. :params req: a swob.Request with an obj in path :raises: HttpException on errors @@ -209,7 +232,7 @@ class StaticLargeObject(object): try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: - return self.app + return self.app(req.environ, start_response) if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge( "Manifest File > %d bytes" % self.max_manifest_size) @@ -230,6 +253,7 @@ class StaticLargeObject(object): if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] + slo_etag = md5() for index, seg_dict in enumerate(parsed_data): obj_path = '/'.join( ['', vrs, account, seg_dict['path'].lstrip('/')]) @@ -260,7 +284,9 @@ class StaticLargeObject(object): total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_path), 'Size Mismatch']) - if seg_dict['etag'] != head_seg_resp.etag: + if seg_dict['etag'] == head_seg_resp.etag: + slo_etag.update(seg_dict['etag']) + else: problem_segments.append([quote(obj_path), 'Etag Mismatch']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified @@ -298,7 +324,9 @@ class StaticLargeObject(object): json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) - return self.app + + slo_context = SloContext(self, slo_etag) + return slo_context.handle_slo_put(req, start_response) def get_segments_to_delete_iter(self, req): """ @@ -376,30 +404,34 @@ class StaticLargeObject(object): out_content_type=out_content_type) return resp - @wsgify - def __call__(self, req): + def __call__(self, env, start_response): """ WSGI entry point """ + req = Request(env) try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: - return self.app - if obj: - if req.method == 'PUT' and \ - req.params.get('multipart-manifest') == 'put': - return self.handle_multipart_put(req) - if req.method == 'DELETE' and \ - req.params.get('multipart-manifest') == 'delete': - return self.handle_multipart_delete(req) - if 'X-Static-Large-Object' in req.headers: - raise HTTPBadRequest( - request=req, - body='X-Static-Large-Object is a reserved header. ' - 'To create a static large object add query param ' - 'multipart-manifest=put.') + return self.app(env, start_response) + try: + if obj: + if req.method == 'PUT' and \ + req.params.get('multipart-manifest') == 'put': + return self.handle_multipart_put(req, start_response) + if req.method == 'DELETE' and \ + req.params.get('multipart-manifest') == 'delete': + return self.handle_multipart_delete(req)(env, + start_response) + if 'X-Static-Large-Object' in req.headers: + raise HTTPBadRequest( + request=req, + body='X-Static-Large-Object is a reserved header. ' + 'To create a static large object add query param ' + 'multipart-manifest=put.') + except HTTPException, err_resp: + return err_resp(env, start_response) - return self.app + return self.app(env, start_response) def filter_factory(global_conf, **local_conf): diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index 206ce9d6b9..1139c7d78f 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -15,6 +15,7 @@ import unittest from mock import patch +from hashlib import md5 from swift.common.middleware import slo from swift.common.utils import json from swift.common.swob import Request, Response, HTTPException @@ -194,7 +195,7 @@ class TestStaticLargeObject(unittest.TestCase): req = Request.blank('/v/a/c/o') req.content_length = self.slo.max_manifest_size + 1 try: - self.slo.handle_multipart_put(req) + self.slo.handle_multipart_put(req, fake_start_response) except HTTPException, e: pass self.assertEquals(e.status_int, 413) @@ -203,7 +204,7 @@ class TestStaticLargeObject(unittest.TestCase): req = Request.blank('/v/a/c/o', body=test_json_data) e = None try: - self.slo.handle_multipart_put(req) + self.slo.handle_multipart_put(req, fake_start_response) except HTTPException, e: pass self.assertEquals(e.status_int, 413) @@ -211,14 +212,14 @@ class TestStaticLargeObject(unittest.TestCase): with patch.object(self.slo, 'min_segment_size', 1000): req = Request.blank('/v/a/c/o', body=test_json_data) try: - self.slo.handle_multipart_put(req) + self.slo.handle_multipart_put(req, fake_start_response) except HTTPException, e: pass self.assertEquals(e.status_int, 400) req = Request.blank('/v/a/c/o', headers={'X-Copy-From': 'lala'}) try: - self.slo.handle_multipart_put(req) + self.slo.handle_multipart_put(req, fake_start_response) except HTTPException, e: pass self.assertEquals(e.status_int, 405) @@ -227,7 +228,9 @@ class TestStaticLargeObject(unittest.TestCase): req = Request.blank( '/?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=test_json_data) - self.assertEquals(self.slo.handle_multipart_put(req), self.app) + self.assertEquals( + self.slo.handle_multipart_put(req, fake_start_response), + ['passed']) def test_handle_multipart_put_success(self): req = Request.blank( @@ -235,7 +238,12 @@ class TestStaticLargeObject(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Accept': 'test'}, body=test_json_data) self.assertTrue('X-Static-Large-Object' not in req.headers) - self.slo(req.environ, fake_start_response) + + def my_fake_start_response(*args, **kwargs): + gen_etag = '"' + md5('etagoftheobjectsegment').hexdigest() + '"' + self.assertTrue(('Etag', gen_etag) in args[1]) + + self.slo(req.environ, my_fake_start_response) self.assertTrue('X-Static-Large-Object' in req.headers) def test_handle_multipart_put_success_allow_small_last_segment(self): @@ -282,7 +290,8 @@ class TestStaticLargeObject(unittest.TestCase): req = Request.blank( '/test_good/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=bad_data) - self.assertRaises(HTTPException, self.slo.handle_multipart_put, req) + self.assertRaises(HTTPException, self.slo.handle_multipart_put, req, + fake_start_response) for bad_data in [ json.dumps([{'path': '/cont', 'etag': 'etagoftheobj', @@ -305,17 +314,17 @@ class TestStaticLargeObject(unittest.TestCase): '/test_good/AUTH_test/c/man?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=bad_data) self.assertRaises(HTTPException, self.slo.handle_multipart_put, - req) + req, fake_start_response) def test_handle_multipart_put_check_data(self): good_data = json.dumps( [{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'}, {'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'}]) req = Request.blank( - '/test_good_check/A/c/man?multipart-manifest=put', + '/test_good_check/A/c/man_3?multipart-manifest=put', environ={'REQUEST_METHOD': 'PUT'}, body=good_data) - self.slo.handle_multipart_put(req) - self.assertEquals(self.app.calls, 2) + self.slo.handle_multipart_put(req, fake_start_response) + self.assertEquals(self.app.calls, 3) self.assert_(req.environ['CONTENT_TYPE'].endswith(';swift_bytes=3')) manifest_data = json.loads(req.environ['wsgi.input'].read()) self.assertEquals(len(manifest_data), 2) @@ -336,7 +345,7 @@ class TestStaticLargeObject(unittest.TestCase): headers={'Accept': 'application/json'}, body=bad_data) try: - self.slo.handle_multipart_put(req) + self.slo.handle_multipart_put(req, fake_start_response) except HTTPException, e: self.assertEquals(self.app.calls, 4) data = json.loads(e.body)