diff --git a/test/functional/test_dlo.py b/test/functional/test_dlo.py new file mode 100644 index 0000000000..36473434bd --- /dev/null +++ b/test/functional/test_dlo.py @@ -0,0 +1,396 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import test.functional as tf +from test.functional.tests import Utils, Base, Base2 +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + + +def setUpModule(): + tf.setup_package() + + +def tearDownModule(): + tf.teardown_package() + + +class TestDloEnv(object): + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + config2 = tf.config.copy() + config2['username'] = tf.config['username3'] + config2['password'] = tf.config['password3'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + cls.container2 = cls.account.container(Utils.create_name()) + + for cont in (cls.container, cls.container2): + if not cont.create(): + raise ResponseError(cls.conn.response) + + # avoid getting a prefix that stops halfway through an encoded + # character + prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") + cls.segment_prefix = prefix + + for letter in ('a', 'b', 'c', 'd', 'e'): + file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter)) + file_item.write(letter * 10) + + file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter)) + file_item.write(letter.upper() * 10) + + for letter in ('f', 'g', 'h', 'i', 'j'): + file_item = cls.container2.file("%s/seg_lower%s" % + (prefix, letter)) + file_item.write(letter * 10) + + man1 = cls.container.file("man1") + man1.write('man1-contents', + hdrs={"X-Object-Manifest": "%s/%s/seg_lower" % + (cls.container.name, prefix)}) + + man2 = cls.container.file("man2") + man2.write('man2-contents', + hdrs={"X-Object-Manifest": "%s/%s/seg_upper" % + (cls.container.name, prefix)}) + + manall = cls.container.file("manall") + manall.write('manall-contents', + hdrs={"X-Object-Manifest": "%s/%s/seg" % + (cls.container.name, prefix)}) + + mancont2 = cls.container.file("mancont2") + mancont2.write( + 'mancont2-contents', + hdrs={"X-Object-Manifest": "%s/%s/seg_lower" % + (cls.container2.name, prefix)}) + + +class TestDlo(Base): + env = TestDloEnv + set_up = False + + def test_get_manifest(self): + file_item = self.env.container.file('man1') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + + file_item = self.env.container.file('man2') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE") + + file_item = self.env.container.file('manall') + file_contents = file_item.read() + self.assertEqual( + file_contents, + ("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" + + "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")) + + def test_get_manifest_document_itself(self): + file_item = self.env.container.file('man1') + file_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(file_contents, "man1-contents") + self.assertEqual(file_item.info()['x_object_manifest'], + "%s/%s/seg_lower" % + (self.env.container.name, self.env.segment_prefix)) + + def test_get_range(self): + file_item = self.env.container.file('man1') + file_contents = file_item.read(size=25, offset=8) + self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd") + + file_contents = file_item.read(size=1, offset=47) + self.assertEqual(file_contents, "e") + + def test_get_range_out_of_range(self): + file_item = self.env.container.file('man1') + + self.assertRaises(ResponseError, file_item.read, size=7, offset=50) + self.assert_status(416) + + def test_copy(self): + # Adding a new segment, copying the manifest, and then deleting the + # segment proves that the new object is really the concatenated + # segments and not just a manifest. + f_segment = self.env.container.file("%s/seg_lowerf" % + (self.env.segment_prefix)) + f_segment.write('ffffffffff') + try: + man1_item = self.env.container.file('man1') + man1_item.copy(self.env.container.name, "copied-man1") + finally: + # try not to leave this around for other tests to stumble over + f_segment.delete() + + file_item = self.env.container.file('copied-man1') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + # The copied object must not have X-Object-Manifest + self.assertNotIn("x_object_manifest", file_item.info()) + + def test_copy_account(self): + # dlo use same account and same container only + acct = self.env.conn.account_name + # Adding a new segment, copying the manifest, and then deleting the + # segment proves that the new object is really the concatenated + # segments and not just a manifest. + f_segment = self.env.container.file("%s/seg_lowerf" % + (self.env.segment_prefix)) + f_segment.write('ffffffffff') + try: + man1_item = self.env.container.file('man1') + man1_item.copy_account(acct, + self.env.container.name, + "copied-man1") + finally: + # try not to leave this around for other tests to stumble over + f_segment.delete() + + file_item = self.env.container.file('copied-man1') + file_contents = file_item.read() + self.assertEqual( + file_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") + # The copied object must not have X-Object-Manifest + self.assertNotIn("x_object_manifest", file_item.info()) + + def test_copy_manifest(self): + # Copying the manifest with multipart-manifest=get query string + # should result in another manifest + try: + man1_item = self.env.container.file('man1') + man1_item.copy(self.env.container.name, "copied-man1", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-man1") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(copied_contents, "man1-contents") + + copied_contents = copied.read() + self.assertEqual( + copied_contents, + "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") + self.assertEqual(man1_item.info()['x_object_manifest'], + copied.info()['x_object_manifest']) + finally: + # try not to leave this around for other tests to stumble over + self.env.container.file("copied-man1").delete() + + def test_dlo_if_match_get(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.read(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_dlo_if_none_match_get(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_dlo_if_match_head(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.info(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_dlo_if_none_match_head(self): + manifest = self.env.container.file("man1") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_dlo_referer_on_segment_container(self): + # First the account2 (test3) should fail + headers = {'X-Auth-Token': self.env.conn2.storage_token, + 'Referer': 'http://blah.example.com'} + dlo_file = self.env.container.file("mancont2") + self.assertRaises(ResponseError, dlo_file.read, + hdrs=headers) + self.assert_status(403) + + # Now set the referer on the dlo container only + referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} + self.env.container.update_metadata(referer_metadata) + + self.assertRaises(ResponseError, dlo_file.read, + hdrs=headers) + self.assert_status(403) + + # Finally set the referer on the segment container + self.env.container2.update_metadata(referer_metadata) + + contents = dlo_file.read(hdrs=headers) + self.assertEqual( + contents, + "ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj") + + def test_dlo_post_with_manifest_header(self): + # verify that performing a POST to a DLO manifest + # preserves the fact that it is a manifest file. + # verify that the x-object-manifest header may be updated. + + # create a new manifest for this test to avoid test coupling. + x_o_m = self.env.container.file('man1').info()['x_object_manifest'] + file_item = self.env.container.file(Utils.create_name()) + file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m}) + + # sanity checks + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('manifest-contents', manifest_contents) + expected_contents = ''.join([(c * 10) for c in 'abcde']) + contents = file_item.read(parms={}) + self.assertEqual(expected_contents, contents) + + # POST a modified x-object-manifest value + new_x_o_m = x_o_m.rstrip('lower') + 'upper' + file_item.post({'x-object-meta-foo': 'bar', + 'x-object-manifest': new_x_o_m}) + + # verify that x-object-manifest was updated + file_item.info() + resp_headers = file_item.conn.response.getheaders() + self.assertIn(('x-object-manifest', new_x_o_m), resp_headers) + self.assertIn(('x-object-meta-foo', 'bar'), resp_headers) + + # verify that manifest content was not changed + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('manifest-contents', manifest_contents) + + # verify that updated manifest points to new content + expected_contents = ''.join([(c * 10) for c in 'ABCDE']) + contents = file_item.read(parms={}) + self.assertEqual(expected_contents, contents) + + # Now revert the manifest to point to original segments, including a + # multipart-manifest=get param just to check that has no effect + file_item.post({'x-object-manifest': x_o_m}, + parms={'multipart-manifest': 'get'}) + + # verify that x-object-manifest was reverted + info = file_item.info() + self.assertIn('x_object_manifest', info) + self.assertEqual(x_o_m, info['x_object_manifest']) + + # verify that manifest content was not changed + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('manifest-contents', manifest_contents) + + # verify that updated manifest points new content + expected_contents = ''.join([(c * 10) for c in 'abcde']) + contents = file_item.read(parms={}) + self.assertEqual(expected_contents, contents) + + def test_dlo_post_without_manifest_header(self): + # verify that a POST to a DLO manifest object with no + # x-object-manifest header will cause the existing x-object-manifest + # header to be lost + + # create a new manifest for this test to avoid test coupling. + x_o_m = self.env.container.file('man1').info()['x_object_manifest'] + file_item = self.env.container.file(Utils.create_name()) + file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m}) + + # sanity checks + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('manifest-contents', manifest_contents) + expected_contents = ''.join([(c * 10) for c in 'abcde']) + contents = file_item.read(parms={}) + self.assertEqual(expected_contents, contents) + + # POST with no x-object-manifest header + file_item.post({}) + + # verify that existing x-object-manifest was removed + info = file_item.info() + self.assertNotIn('x_object_manifest', info) + + # verify that object content was not changed + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('manifest-contents', manifest_contents) + + # verify that object is no longer a manifest + contents = file_item.read(parms={}) + self.assertEqual('manifest-contents', contents) + + def test_dlo_post_with_manifest_regular_object(self): + # verify that performing a POST to a regular object + # with a manifest header will create a DLO. + + # Put a regular object + file_item = self.env.container.file(Utils.create_name()) + file_item.write('file contents', hdrs={}) + + # sanity checks + file_contents = file_item.read(parms={}) + self.assertEqual('file contents', file_contents) + + # get the path associated with man1 + x_o_m = self.env.container.file('man1').info()['x_object_manifest'] + + # POST a x-object-manifest value to the regular object + file_item.post({'x-object-manifest': x_o_m}) + + # verify that the file is now a manifest + manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) + self.assertEqual('file contents', manifest_contents) + expected_contents = ''.join([(c * 10) for c in 'abcde']) + contents = file_item.read(parms={}) + self.assertEqual(expected_contents, contents) + file_item.info() + resp_headers = file_item.conn.response.getheaders() + self.assertIn(('x-object-manifest', x_o_m), resp_headers) + + +class TestDloUTF8(Base2, TestDlo): + set_up = False diff --git a/test/functional/test_slo.py b/test/functional/test_slo.py new file mode 100644 index 0000000000..b4ced42d4b --- /dev/null +++ b/test/functional/test_slo.py @@ -0,0 +1,938 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import email.parser +import hashlib +import itertools +import json +from copy import deepcopy +from unittest2 import SkipTest + +import test.functional as tf +from test.functional import cluster_info +from test.functional.tests import Utils, Base, Base2 +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + + +def setUpModule(): + tf.setup_package() + + +def tearDownModule(): + tf.teardown_package() + + +class TestSloEnv(object): + slo_enabled = None # tri-state: None initially, then True/False + + @classmethod + def create_segments(cls, container): + seg_info = {} + for letter, size in (('a', 1024 * 1024), + ('b', 1024 * 1024), + ('c', 1024 * 1024), + ('d', 1024 * 1024), + ('e', 1)): + seg_name = "seg_%s" % letter + file_item = container.file(seg_name) + file_item.write(letter * size) + seg_info[seg_name] = { + 'size_bytes': size, + 'etag': file_item.md5, + 'path': '/%s/%s' % (container.name, seg_name)} + return seg_info + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account2 = cls.conn2.get_account() + cls.account2.delete_containers() + config3 = tf.config.copy() + config3['username'] = tf.config['username3'] + config3['password'] = tf.config['password3'] + cls.conn3 = Connection(config3) + cls.conn3.authenticate() + + if cls.slo_enabled is None: + cls.slo_enabled = 'slo' in cluster_info + if not cls.slo_enabled: + return + + cls.account = Account(cls.conn, tf.config.get('account', + tf.config['username'])) + cls.account.delete_containers() + + cls.container = cls.account.container(Utils.create_name()) + cls.container2 = cls.account.container(Utils.create_name()) + + for cont in (cls.container, cls.container2): + if not cont.create(): + raise ResponseError(cls.conn.response) + + cls.seg_info = seg_info = cls.create_segments(cls.container) + + file_item = cls.container.file("manifest-abcde") + file_item.write( + json.dumps([seg_info['seg_a'], seg_info['seg_b'], + seg_info['seg_c'], seg_info['seg_d'], + seg_info['seg_e']]), + parms={'multipart-manifest': 'put'}) + + # Put the same manifest in the container2 + file_item = cls.container2.file("manifest-abcde") + file_item.write( + json.dumps([seg_info['seg_a'], seg_info['seg_b'], + seg_info['seg_c'], seg_info['seg_d'], + seg_info['seg_e']]), + parms={'multipart-manifest': 'put'}) + + file_item = cls.container.file('manifest-cd') + cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']]) + file_item.write(cd_json, parms={'multipart-manifest': 'put'}) + cd_etag = hashlib.md5(seg_info['seg_c']['etag'] + + seg_info['seg_d']['etag']).hexdigest() + + file_item = cls.container.file("manifest-bcd-submanifest") + file_item.write( + json.dumps([seg_info['seg_b'], + {'etag': cd_etag, + 'size_bytes': (seg_info['seg_c']['size_bytes'] + + seg_info['seg_d']['size_bytes']), + 'path': '/%s/%s' % (cls.container.name, + 'manifest-cd')}]), + parms={'multipart-manifest': 'put'}) + bcd_submanifest_etag = hashlib.md5( + seg_info['seg_b']['etag'] + cd_etag).hexdigest() + + file_item = cls.container.file("manifest-abcde-submanifest") + file_item.write( + json.dumps([ + seg_info['seg_a'], + {'etag': bcd_submanifest_etag, + 'size_bytes': (seg_info['seg_b']['size_bytes'] + + seg_info['seg_c']['size_bytes'] + + seg_info['seg_d']['size_bytes']), + 'path': '/%s/%s' % (cls.container.name, + 'manifest-bcd-submanifest')}, + seg_info['seg_e']]), + parms={'multipart-manifest': 'put'}) + abcde_submanifest_etag = hashlib.md5( + seg_info['seg_a']['etag'] + bcd_submanifest_etag + + seg_info['seg_e']['etag']).hexdigest() + abcde_submanifest_size = (seg_info['seg_a']['size_bytes'] + + seg_info['seg_b']['size_bytes'] + + seg_info['seg_c']['size_bytes'] + + seg_info['seg_d']['size_bytes'] + + seg_info['seg_e']['size_bytes']) + + file_item = cls.container.file("ranged-manifest") + file_item.write( + json.dumps([ + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '-1048578'}, # 'c' + ('d' * 2**20) + 'e' + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '524288-1572863'}, # 'a' * 2**19 + 'b' * 2**19 + {'etag': abcde_submanifest_etag, + 'size_bytes': abcde_submanifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'manifest-abcde-submanifest'), + 'range': '3145727-3145728'}]), # 'cd' + parms={'multipart-manifest': 'put'}) + ranged_manifest_etag = hashlib.md5( + abcde_submanifest_etag + ':3145727-4194304;' + + abcde_submanifest_etag + ':524288-1572863;' + + abcde_submanifest_etag + ':3145727-3145728;').hexdigest() + ranged_manifest_size = 2 * 1024 * 1024 + 4 + + file_item = cls.container.file("ranged-submanifest") + file_item.write( + json.dumps([ + seg_info['seg_c'], + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest')}, + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest'), + 'range': '524289-1572865'}, + {'etag': ranged_manifest_etag, + 'size_bytes': ranged_manifest_size, + 'path': '/%s/%s' % (cls.container.name, + 'ranged-manifest'), + 'range': '-3'}]), + parms={'multipart-manifest': 'put'}) + + file_item = cls.container.file("manifest-db") + file_item.write( + json.dumps([ + {'path': seg_info['seg_d']['path'], 'etag': None, + 'size_bytes': None}, + {'path': seg_info['seg_b']['path'], 'etag': None, + 'size_bytes': None}, + ]), parms={'multipart-manifest': 'put'}) + + file_item = cls.container.file("ranged-manifest-repeated-segment") + file_item.write( + json.dumps([ + {'path': seg_info['seg_a']['path'], 'etag': None, + 'size_bytes': None, 'range': '-1048578'}, + {'path': seg_info['seg_a']['path'], 'etag': None, + 'size_bytes': None}, + {'path': seg_info['seg_b']['path'], 'etag': None, + 'size_bytes': None, 'range': '-1048578'}, + ]), parms={'multipart-manifest': 'put'}) + + +class TestSlo(Base): + env = TestSloEnv + set_up = False + + def setUp(self): + super(TestSlo, self).setUp() + if self.env.slo_enabled is False: + raise SkipTest("SLO not enabled") + elif self.env.slo_enabled is not True: + # just some sanity checking + raise Exception( + "Expected slo_enabled to be True/False, got %r" % + (self.env.slo_enabled,)) + + def test_slo_get_simple_manifest(self): + file_item = self.env.container.file('manifest-abcde') + file_contents = file_item.read() + self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) + self.assertEqual('a', file_contents[0]) + self.assertEqual('a', file_contents[1024 * 1024 - 1]) + self.assertEqual('b', file_contents[1024 * 1024]) + self.assertEqual('d', file_contents[-2]) + self.assertEqual('e', file_contents[-1]) + + def test_slo_container_listing(self): + # the listing object size should equal the sum of the size of the + # segments, not the size of the manifest body + file_item = self.env.container.file(Utils.create_name()) + file_item.write( + json.dumps([self.env.seg_info['seg_a']]), + parms={'multipart-manifest': 'put'}) + # The container listing has the etag of the actual manifest object + # contents which we get using multipart-manifest=get. Arguably this + # should be the etag that we get when NOT using multipart-manifest=get, + # to be consistent with size and content-type. But here we at least + # verify that it remains consistent when the object is updated with a + # POST. + file_item.initialize(parms={'multipart-manifest': 'get'}) + expected_etag = file_item.etag + + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual('application/octet-stream', + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + # now POST updated content-type file + file_item.content_type = 'image/jpeg' + file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}) + file_item.initialize() + self.assertEqual('image/jpeg', file_item.content_type) # sanity + + # verify that the container listing is consistent with the file + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual(file_item.content_type, + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + # now POST with no change to content-type + file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}, + cfg={'no_content_type': True}) + file_item.initialize() + self.assertEqual('image/jpeg', file_item.content_type) # sanity + + # verify that the container listing is consistent with the file + listing = self.env.container.files(parms={'format': 'json'}) + for f_dict in listing: + if f_dict['name'] == file_item.name: + self.assertEqual(1024 * 1024, f_dict['bytes']) + self.assertEqual(file_item.content_type, + f_dict['content_type']) + self.assertEqual(expected_etag, f_dict['hash']) + break + else: + self.fail('Failed to find manifest file in container listing') + + def test_slo_get_nested_manifest(self): + file_item = self.env.container.file('manifest-abcde-submanifest') + file_contents = file_item.read() + self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) + self.assertEqual('a', file_contents[0]) + self.assertEqual('a', file_contents[1024 * 1024 - 1]) + self.assertEqual('b', file_contents[1024 * 1024]) + self.assertEqual('d', file_contents[-2]) + self.assertEqual('e', file_contents[-1]) + + def test_slo_get_ranged_manifest(self): + file_item = self.env.container.file('ranged-manifest') + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(file_item.read())] + self.assertEqual([ + ('c', 1), + ('d', 1024 * 1024), + ('e', 1), + ('a', 512 * 1024), + ('b', 512 * 1024), + ('c', 1), + ('d', 1)], grouped_file_contents) + + def test_slo_get_ranged_manifest_repeated_segment(self): + file_item = self.env.container.file('ranged-manifest-repeated-segment') + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(file_item.read())] + self.assertEqual( + [('a', 2097152), ('b', 1048576)], + grouped_file_contents) + + def test_slo_get_ranged_submanifest(self): + file_item = self.env.container.file('ranged-submanifest') + grouped_file_contents = [ + (char, sum(1 for _char in grp)) + for char, grp in itertools.groupby(file_item.read())] + self.assertEqual([ + ('c', 1024 * 1024 + 1), + ('d', 1024 * 1024), + ('e', 1), + ('a', 512 * 1024), + ('b', 512 * 1024), + ('c', 1), + ('d', 512 * 1024 + 1), + ('e', 1), + ('a', 512 * 1024), + ('b', 1), + ('c', 1), + ('d', 1)], grouped_file_contents) + + def test_slo_ranged_get(self): + file_item = self.env.container.file('manifest-abcde') + file_contents = file_item.read(size=1024 * 1024 + 2, + offset=1024 * 1024 - 1) + self.assertEqual('a', file_contents[0]) + self.assertEqual('b', file_contents[1]) + self.assertEqual('b', file_contents[-2]) + self.assertEqual('c', file_contents[-1]) + + def test_slo_multi_ranged_get(self): + file_item = self.env.container.file('manifest-abcde') + file_contents = file_item.read( + hdrs={"Range": "bytes=1048571-1048580,2097147-2097156"}) + + # See testMultiRangeGets for explanation + parser = email.parser.FeedParser() + parser.feed("Content-Type: %s\r\n\r\n" % file_item.content_type) + parser.feed(file_contents) + + root_message = parser.close() + self.assertTrue(root_message.is_multipart()) # sanity check + + byteranges = root_message.get_payload() + self.assertEqual(len(byteranges), 2) + + self.assertEqual(byteranges[0]['Content-Type'], + "application/octet-stream") + self.assertEqual( + byteranges[0]['Content-Range'], "bytes 1048571-1048580/4194305") + self.assertEqual(byteranges[0].get_payload(), "aaaaabbbbb") + + self.assertEqual(byteranges[1]['Content-Type'], + "application/octet-stream") + self.assertEqual( + byteranges[1]['Content-Range'], "bytes 2097147-2097156/4194305") + self.assertEqual(byteranges[1].get_payload(), "bbbbbccccc") + + def test_slo_ranged_submanifest(self): + file_item = self.env.container.file('manifest-abcde-submanifest') + file_contents = file_item.read(size=1024 * 1024 + 2, + offset=1024 * 1024 * 2 - 1) + self.assertEqual('b', file_contents[0]) + self.assertEqual('c', file_contents[1]) + self.assertEqual('c', file_contents[-2]) + self.assertEqual('d', file_contents[-1]) + + def test_slo_etag_is_hash_of_etags(self): + expected_hash = hashlib.md5() + expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest()) + expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest()) + expected_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest()) + expected_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest()) + expected_hash.update(hashlib.md5('e').hexdigest()) + expected_etag = expected_hash.hexdigest() + + file_item = self.env.container.file('manifest-abcde') + self.assertEqual(expected_etag, file_item.info()['etag']) + + def test_slo_etag_is_hash_of_etags_submanifests(self): + + def hd(x): + return hashlib.md5(x).hexdigest() + + expected_etag = hd(hd('a' * 1024 * 1024) + + hd(hd('b' * 1024 * 1024) + + hd(hd('c' * 1024 * 1024) + + hd('d' * 1024 * 1024))) + + hd('e')) + + file_item = self.env.container.file('manifest-abcde-submanifest') + self.assertEqual(expected_etag, file_item.info()['etag']) + + def test_slo_etag_mismatch(self): + file_item = self.env.container.file("manifest-a-bad-etag") + try: + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024, + 'etag': 'not it', + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_size_mismatch(self): + file_item = self.env.container.file("manifest-a-bad-size") + try: + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024 - 1, + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_unspecified_etag(self): + file_item = self.env.container.file("manifest-a-unspecified-etag") + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024, + 'etag': None, + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + self.assert_status(201) + + def test_slo_unspecified_size(self): + file_item = self.env.container.file("manifest-a-unspecified-size") + file_item.write( + json.dumps([{ + 'size_bytes': None, + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + self.assert_status(201) + + def test_slo_missing_etag(self): + file_item = self.env.container.file("manifest-a-missing-etag") + try: + file_item.write( + json.dumps([{ + 'size_bytes': 1024 * 1024, + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_missing_size(self): + file_item = self.env.container.file("manifest-a-missing-size") + try: + file_item.write( + json.dumps([{ + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), + parms={'multipart-manifest': 'put'}) + except ResponseError as err: + self.assertEqual(400, err.status) + else: + self.fail("Expected ResponseError but didn't get it") + + def test_slo_overwrite_segment_with_manifest(self): + file_item = self.env.container.file("seg_b") + with self.assertRaises(ResponseError) as catcher: + file_item.write( + json.dumps([ + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}, + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('b' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_b')}, + {'size_bytes': 1024 * 1024, + 'etag': hashlib.md5('c' * 1024 * 1024).hexdigest(), + 'path': '/%s/%s' % (self.env.container.name, 'seg_c')}]), + parms={'multipart-manifest': 'put'}) + self.assertEqual(400, catcher.exception.status) + + def test_slo_copy(self): + file_item = self.env.container.file("manifest-abcde") + file_item.copy(self.env.container.name, "copied-abcde") + + copied = self.env.container.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + + def test_slo_copy_account(self): + acct = self.env.conn.account_name + # same account copy + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, self.env.container.name, "copied-abcde") + + copied = self.env.container.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + + # copy to different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, dest_cont, "copied-abcde") + + copied = dest_cont.file("copied-abcde") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) + + def test_slo_copy_the_manifest(self): + source = self.env.container.file("manifest-abcde") + source_contents = source.read(parms={'multipart-manifest': 'get'}) + source_json = json.loads(source_contents) + source.initialize() + self.assertEqual('application/octet-stream', source.content_type) + source.initialize(parms={'multipart-manifest': 'get'}) + source_hash = hashlib.md5() + source_hash.update(source_contents) + self.assertEqual(source_hash.hexdigest(), source.etag) + + self.assertTrue(source.copy(self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) + + copied = self.env.container.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + copied_json = json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + self.assertEqual(source_json, copied_json) + copied.initialize() + self.assertEqual('application/octet-stream', copied.content_type) + copied.initialize(parms={'multipart-manifest': 'get'}) + copied_hash = hashlib.md5() + copied_hash.update(copied_contents) + self.assertEqual(copied_hash.hexdigest(), copied.etag) + + # verify the listing metadata + listing = self.env.container.files(parms={'format': 'json'}) + names = {} + for f_dict in listing: + if f_dict['name'] in ('manifest-abcde', + 'copied-abcde-manifest-only'): + names[f_dict['name']] = f_dict + + self.assertIn('manifest-abcde', names) + actual = names['manifest-abcde'] + self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) + self.assertEqual('application/octet-stream', actual['content_type']) + self.assertEqual(source.etag, actual['hash']) + + self.assertIn('copied-abcde-manifest-only', names) + actual = names['copied-abcde-manifest-only'] + self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) + self.assertEqual('application/octet-stream', actual['content_type']) + self.assertEqual(copied.etag, actual['hash']) + + def test_slo_copy_the_manifest_updating_metadata(self): + source = self.env.container.file("manifest-abcde") + source.content_type = 'application/octet-stream' + source.sync_metadata({'test': 'original'}) + source_contents = source.read(parms={'multipart-manifest': 'get'}) + source_json = json.loads(source_contents) + source.initialize() + self.assertEqual('application/octet-stream', source.content_type) + source.initialize(parms={'multipart-manifest': 'get'}) + source_hash = hashlib.md5() + source_hash.update(source_contents) + self.assertEqual(source_hash.hexdigest(), source.etag) + self.assertEqual(source.metadata['test'], 'original') + + self.assertTrue( + source.copy(self.env.container.name, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}, + hdrs={'Content-Type': 'image/jpeg', + 'X-Object-Meta-Test': 'updated'})) + + copied = self.env.container.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + copied_json = json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + self.assertEqual(source_json, copied_json) + copied.initialize() + self.assertEqual('image/jpeg', copied.content_type) + copied.initialize(parms={'multipart-manifest': 'get'}) + copied_hash = hashlib.md5() + copied_hash.update(copied_contents) + self.assertEqual(copied_hash.hexdigest(), copied.etag) + self.assertEqual(copied.metadata['test'], 'updated') + + # verify the listing metadata + listing = self.env.container.files(parms={'format': 'json'}) + names = {} + for f_dict in listing: + if f_dict['name'] in ('manifest-abcde', + 'copied-abcde-manifest-only'): + names[f_dict['name']] = f_dict + + self.assertIn('manifest-abcde', names) + actual = names['manifest-abcde'] + self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) + self.assertEqual('application/octet-stream', actual['content_type']) + # the container listing should have the etag of the manifest contents + self.assertEqual(source.etag, actual['hash']) + + self.assertIn('copied-abcde-manifest-only', names) + actual = names['copied-abcde-manifest-only'] + self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) + self.assertEqual('image/jpeg', actual['content_type']) + self.assertEqual(copied.etag, actual['hash']) + + def test_slo_copy_the_manifest_account(self): + acct = self.env.conn.account_name + # same account + file_item = self.env.container.file("manifest-abcde") + file_item.copy_account(acct, + self.env.container.name, + "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + + copied = self.env.container.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + + # different account + acct = self.env.conn2.account_name + dest_cont = self.env.account2.container(Utils.create_name()) + self.assertTrue(dest_cont.create(hdrs={ + 'X-Container-Write': self.env.conn.user_acl + })) + + # manifest copy will fail because there is no read access to segments + # in destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('403 Forbidden'), + 'Unexpected response body %r' % resp_body) + + # create segments container in account2 with read access for account1 + segs_container = self.env.account2.container(self.env.container.name) + self.assertTrue(segs_container.create(hdrs={ + 'X-Container-Read': self.env.conn.user_acl + })) + + # manifest copy will still fail because there are no segments in + # destination account + file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'}) + self.assertEqual(400, file_item.conn.response.status) + resp_body = file_item.conn.response.read() + self.assertEqual(5, resp_body.count('404 Not Found'), + 'Unexpected response body %r' % resp_body) + + # create segments in account2 container with same name as in account1, + # manifest copy now succeeds + self.env.create_segments(segs_container) + + self.assertTrue(file_item.copy_account( + acct, dest_cont, "copied-abcde-manifest-only", + parms={'multipart-manifest': 'get'})) + + copied = dest_cont.file("copied-abcde-manifest-only") + copied_contents = copied.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(copied_contents) + except ValueError: + self.fail("COPY didn't copy the manifest (invalid json on GET)") + + def _make_manifest(self): + file_item = self.env.container.file("manifest-post") + seg_info = self.env.seg_info + file_item.write( + json.dumps([seg_info['seg_a'], seg_info['seg_b'], + seg_info['seg_c'], seg_info['seg_d'], + seg_info['seg_e']]), + parms={'multipart-manifest': 'put'}) + return file_item + + def test_slo_post_the_manifest_metadata_update(self): + file_item = self._make_manifest() + # sanity check, check the object is an SLO manifest + file_item.info() + file_item.header_fields([('slo', 'x-static-large-object')]) + + # POST a user metadata (i.e. x-object-meta-post) + file_item.sync_metadata({'post': 'update'}) + + updated = self.env.container.file("manifest-post") + updated.info() + updated.header_fields([('user-meta', 'x-object-meta-post')]) # sanity + updated.header_fields([('slo', 'x-static-large-object')]) + updated_contents = updated.read(parms={'multipart-manifest': 'get'}) + try: + json.loads(updated_contents) + except ValueError: + self.fail("Unexpected content on GET, expected a json body") + + def test_slo_post_the_manifest_metadata_update_with_qs(self): + # multipart-manifest query should be ignored on post + for verb in ('put', 'get', 'delete'): + file_item = self._make_manifest() + # sanity check, check the object is an SLO manifest + file_item.info() + file_item.header_fields([('slo', 'x-static-large-object')]) + # POST a user metadata (i.e. x-object-meta-post) + file_item.sync_metadata(metadata={'post': 'update'}, + parms={'multipart-manifest': verb}) + updated = self.env.container.file("manifest-post") + updated.info() + updated.header_fields( + [('user-meta', 'x-object-meta-post')]) # sanity + updated.header_fields([('slo', 'x-static-large-object')]) + updated_contents = updated.read( + parms={'multipart-manifest': 'get'}) + try: + json.loads(updated_contents) + except ValueError: + self.fail( + "Unexpected content on GET, expected a json body") + + def test_slo_get_the_manifest(self): + manifest = self.env.container.file("manifest-abcde") + got_body = manifest.read(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + manifest.content_type) + try: + json.loads(got_body) + except ValueError: + self.fail("GET with multipart-manifest=get got invalid json") + + def test_slo_get_the_manifest_with_details_from_server(self): + manifest = self.env.container.file("manifest-db") + got_body = manifest.read(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + manifest.content_type) + try: + value = json.loads(got_body) + except ValueError: + self.fail("GET with multipart-manifest=get got invalid json") + + self.assertEqual(len(value), 2) + self.assertEqual(value[0]['bytes'], 1024 * 1024) + self.assertEqual(value[0]['hash'], + hashlib.md5('d' * 1024 * 1024).hexdigest()) + self.assertEqual(value[0]['name'], + '/%s/seg_d' % self.env.container.name.decode("utf-8")) + + self.assertEqual(value[1]['bytes'], 1024 * 1024) + self.assertEqual(value[1]['hash'], + hashlib.md5('b' * 1024 * 1024).hexdigest()) + self.assertEqual(value[1]['name'], + '/%s/seg_b' % self.env.container.name.decode("utf-8")) + + def test_slo_get_raw_the_manifest_with_details_from_server(self): + manifest = self.env.container.file("manifest-db") + got_body = manifest.read(parms={'multipart-manifest': 'get', + 'format': 'raw'}) + + # raw format should have the actual manifest object content-type + self.assertEqual('application/octet-stream', manifest.content_type) + try: + value = json.loads(got_body) + except ValueError: + msg = "GET with multipart-manifest=get&format=raw got invalid json" + self.fail(msg) + + self.assertEqual( + set(value[0].keys()), set(('size_bytes', 'etag', 'path'))) + self.assertEqual(len(value), 2) + self.assertEqual(value[0]['size_bytes'], 1024 * 1024) + self.assertEqual(value[0]['etag'], + hashlib.md5('d' * 1024 * 1024).hexdigest()) + self.assertEqual(value[0]['path'], + '/%s/seg_d' % self.env.container.name.decode("utf-8")) + self.assertEqual(value[1]['size_bytes'], 1024 * 1024) + self.assertEqual(value[1]['etag'], + hashlib.md5('b' * 1024 * 1024).hexdigest()) + self.assertEqual(value[1]['path'], + '/%s/seg_b' % self.env.container.name.decode("utf-8")) + + file_item = self.env.container.file("manifest-from-get-raw") + file_item.write(got_body, parms={'multipart-manifest': 'put'}) + + file_contents = file_item.read() + self.assertEqual(2 * 1024 * 1024, len(file_contents)) + + def test_slo_head_the_manifest(self): + manifest = self.env.container.file("manifest-abcde") + got_info = manifest.info(parms={'multipart-manifest': 'get'}) + + self.assertEqual('application/json; charset=utf-8', + got_info['content_type']) + + def test_slo_if_match_get(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.read(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_slo_if_none_match_put(self): + file_item = self.env.container.file("manifest-if-none-match") + manifest = json.dumps([{ + 'size_bytes': 1024 * 1024, + 'etag': None, + 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]) + + self.assertRaises(ResponseError, file_item.write, manifest, + parms={'multipart-manifest': 'put'}, + hdrs={'If-None-Match': '"not-star"'}) + self.assert_status(400) + + file_item.write(manifest, parms={'multipart-manifest': 'put'}, + hdrs={'If-None-Match': '*'}) + self.assert_status(201) + + self.assertRaises(ResponseError, file_item.write, manifest, + parms={'multipart-manifest': 'put'}, + hdrs={'If-None-Match': '*'}) + self.assert_status(412) + + def test_slo_if_none_match_get(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.read, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_slo_if_match_head(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-Match': 'not-%s' % etag}) + self.assert_status(412) + + manifest.info(hdrs={'If-Match': etag}) + self.assert_status(200) + + def test_slo_if_none_match_head(self): + manifest = self.env.container.file("manifest-abcde") + etag = manifest.info()['etag'] + + self.assertRaises(ResponseError, manifest.info, + hdrs={'If-None-Match': etag}) + self.assert_status(304) + + manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) + self.assert_status(200) + + def test_slo_referer_on_segment_container(self): + # First the account2 (test3) should fail + headers = {'X-Auth-Token': self.env.conn3.storage_token, + 'Referer': 'http://blah.example.com'} + slo_file = self.env.container2.file('manifest-abcde') + self.assertRaises(ResponseError, slo_file.read, + hdrs=headers) + self.assert_status(403) + + # Now set the referer on the slo container only + referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} + self.env.container2.update_metadata(referer_metadata) + + self.assertRaises(ResponseError, slo_file.read, + hdrs=headers) + self.assert_status(409) + + # Finally set the referer on the segment container + self.env.container.update_metadata(referer_metadata) + contents = slo_file.read(hdrs=headers) + self.assertEqual(4 * 1024 * 1024 + 1, len(contents)) + self.assertEqual('a', contents[0]) + self.assertEqual('a', contents[1024 * 1024 - 1]) + self.assertEqual('b', contents[1024 * 1024]) + self.assertEqual('d', contents[-2]) + self.assertEqual('e', contents[-1]) + + +class TestSloUTF8(Base2, TestSlo): + set_up = False diff --git a/test/functional/test_tempurl.py b/test/functional/test_tempurl.py new file mode 100644 index 0000000000..a4e298d0e8 --- /dev/null +++ b/test/functional/test_tempurl.py @@ -0,0 +1,674 @@ +#!/usr/bin/python -u +# Copyright (c) 2010-2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hmac +import hashlib +import json +import time +from copy import deepcopy +from six.moves import urllib +from unittest2 import SkipTest + +import test.functional as tf +from test.functional import cluster_info +from test.functional.tests import Utils, Base, Base2 +from test.functional import requires_acls +from test.functional.swift_test_client import Account, Connection, \ + ResponseError + + +def setUpModule(): + tf.setup_package() + + +def tearDownModule(): + tf.teardown_package() + + +class TestTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, tf.config.get('account', tf.config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({ + 'temp-url-key': cls.tempurl_key, + 'temp-url-key-2': cls.tempurl_key2 + }) + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create(): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestTempurl(Base): + env = TestTempurlEnv + set_up = False + + def setUp(self): + super(TestTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + seg1 = self.env.container.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + container2 = self.env.account.container(Utils.create_name()) + container2.create() + + manifest = container2.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl works fine for account tempurl key + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + self.assert_status([200]) + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assertTrue(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_PUT_manifest_access(self): + new_obj = self.env.container.file(Utils.create_name()) + + # give out a signature which allows a PUT to new_obj + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # try to create manifest pointing to some random container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % 'some_random_container' + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + # create some other container + other_container = self.env.account.container(Utils.create_name()) + if not other_container.create(): + raise ResponseError(self.conn.response) + + # try to create manifest pointing to new container + try: + new_obj.write('', { + 'x-object-manifest': '%s/foo' % other_container + }, parms=put_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + # try again using a tempurl POST to an already created object + new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True}) + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'POST', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + post_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + try: + new_obj.post({'x-object-manifest': '%s/foo' % other_container}, + parms=post_parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 400) + else: + self.fail('request did not error') + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assertTrue(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + +class TestTempurlUTF8(Base2, TestTempurl): + set_up = False + + +class TestContainerTempurlEnv(object): + tempurl_enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + if cls.tempurl_enabled is None: + cls.tempurl_enabled = 'tempurl' in cluster_info + if not cls.tempurl_enabled: + return + + cls.tempurl_key = Utils.create_name() + cls.tempurl_key2 = Utils.create_name() + + cls.account = Account( + cls.conn, tf.config.get('account', tf.config['username'])) + cls.account.delete_containers() + + # creating another account and connection + # for ACL tests + config2 = deepcopy(tf.config) + config2['account'] = tf.config['account2'] + config2['username'] = tf.config['username2'] + config2['password'] = tf.config['password2'] + cls.conn2 = Connection(config2) + cls.conn2.authenticate() + cls.account2 = Account( + cls.conn2, config2.get('account', config2['username'])) + cls.account2 = cls.conn2.get_account() + + cls.container = cls.account.container(Utils.create_name()) + if not cls.container.create({ + 'x-container-meta-temp-url-key': cls.tempurl_key, + 'x-container-meta-temp-url-key-2': cls.tempurl_key2, + 'x-container-read': cls.account2.name}): + raise ResponseError(cls.conn.response) + + cls.obj = cls.container.file(Utils.create_name()) + cls.obj.write("obj contents") + cls.other_obj = cls.container.file(Utils.create_name()) + cls.other_obj.write("other obj contents") + + +class TestContainerTempurl(Base): + env = TestContainerTempurlEnv + set_up = False + + def setUp(self): + super(TestContainerTempurl, self).setUp() + if self.env.tempurl_enabled is False: + raise SkipTest("TempURL not enabled") + elif self.env.tempurl_enabled is not True: + # just some sanity checking + raise Exception( + "Expected tempurl_enabled to be True/False, got %r" % + (self.env.tempurl_enabled,)) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + self.obj_tempurl_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + # GET tempurls also allow HEAD requests + self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True})) + + def test_GET_with_key_2(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key2) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + def test_PUT(self): + new_obj = self.env.container.file(Utils.create_name()) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'PUT', expires, self.env.conn.make_path(new_obj.path), + self.env.tempurl_key) + put_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + new_obj.write('new obj contents', + parms=put_parms, cfg={'no_auth_token': True}) + self.assertEqual(new_obj.read(), "new obj contents") + + # PUT tempurls also allow HEAD requests + self.assertTrue(new_obj.info(parms=put_parms, + cfg={'no_auth_token': True})) + + def test_HEAD(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), + self.env.tempurl_key) + head_parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + self.assertTrue(self.env.obj.info(parms=head_parms, + cfg={'no_auth_token': True})) + # HEAD tempurls don't allow PUT or GET requests, despite the fact that + # PUT and GET tempurls both allow HEAD requests + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + self.assertRaises(ResponseError, self.env.other_obj.write, + 'new contents', + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_different_object(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + self.assertRaises(ResponseError, self.env.other_obj.read, + cfg={'no_auth_token': True}, + parms=self.obj_tempurl_parms) + self.assert_status([401]) + + def test_changing_sig(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_sig'][0] == 'a': + parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] + else: + parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + def test_changing_expires(self): + contents = self.env.obj.read( + parms=self.obj_tempurl_parms, + cfg={'no_auth_token': True}) + self.assertEqual(contents, "obj contents") + + parms = self.obj_tempurl_parms.copy() + if parms['temp_url_expires'][-1] == '0': + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' + else: + parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' + + self.assertRaises(ResponseError, self.env.obj.read, + cfg={'no_auth_token': True}, + parms=parms) + self.assert_status([401]) + + @requires_acls + def test_tempurl_keys_visible_to_account_owner(self): + if not tf.cluster_info.get('tempauth'): + raise SkipTest('TEMP AUTH SPECIFIC TEST') + metadata = self.env.container.info() + self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key) + self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2) + + @requires_acls + def test_tempurl_keys_hidden_from_acl_readonly(self): + if not tf.cluster_info.get('tempauth'): + raise SkipTest('TEMP AUTH SPECIFIC TEST') + original_token = self.env.container.conn.storage_token + self.env.container.conn.storage_token = self.env.conn2.storage_token + metadata = self.env.container.info() + self.env.container.conn.storage_token = original_token + + self.assertNotIn( + 'tempurl_key', metadata, + 'Container TempURL key found, should not be visible ' + 'to readonly ACLs') + self.assertNotIn( + 'tempurl_key2', metadata, + 'Container TempURL key-2 found, should not be visible ' + 'to readonly ACLs') + + def test_GET_DLO_inside_container(self): + seg1 = self.env.container.file( + "get-dlo-inside-seg1" + Utils.create_name()) + seg2 = self.env.container.file( + "get-dlo-inside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % + (self.env.container.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) + self.assertEqual(contents, "one fish two fish red fish blue fish") + + def test_GET_DLO_outside_container(self): + container2 = self.env.account.container(Utils.create_name()) + container2.create() + seg1 = container2.file( + "get-dlo-outside-seg1" + Utils.create_name()) + seg2 = container2.file( + "get-dlo-outside-seg2" + Utils.create_name()) + seg1.write("one fish two fish ") + seg2.write("red fish blue fish") + + manifest = self.env.container.file("manifest" + Utils.create_name()) + manifest.write( + '', + hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % + (container2.name,)}) + + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, + 'temp_url_expires': str(expires)} + + # cross container tempurl does not work for container tempurl key + try: + manifest.read(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + try: + manifest.info(parms=parms, cfg={'no_auth_token': True}) + except ResponseError as e: + self.assertEqual(e.status, 401) + else: + self.fail('request did not error') + + +class TestContainerTempurlUTF8(Base2, TestContainerTempurl): + set_up = False + + +class TestSloTempurlEnv(object): + enabled = None # tri-state: None initially, then True/False + + @classmethod + def setUp(cls): + cls.conn = Connection(tf.config) + cls.conn.authenticate() + + if cls.enabled is None: + cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info + + cls.tempurl_key = Utils.create_name() + + cls.account = Account( + cls.conn, tf.config.get('account', tf.config['username'])) + cls.account.delete_containers() + cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) + + cls.manifest_container = cls.account.container(Utils.create_name()) + cls.segments_container = cls.account.container(Utils.create_name()) + if not cls.manifest_container.create(): + raise ResponseError(cls.conn.response) + if not cls.segments_container.create(): + raise ResponseError(cls.conn.response) + + seg1 = cls.segments_container.file(Utils.create_name()) + seg1.write('1' * 1024 * 1024) + + seg2 = cls.segments_container.file(Utils.create_name()) + seg2.write('2' * 1024 * 1024) + + cls.manifest_data = [{'size_bytes': 1024 * 1024, + 'etag': seg1.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg1.name)}, + {'size_bytes': 1024 * 1024, + 'etag': seg2.md5, + 'path': '/%s/%s' % (cls.segments_container.name, + seg2.name)}] + + cls.manifest = cls.manifest_container.file(Utils.create_name()) + cls.manifest.write( + json.dumps(cls.manifest_data), + parms={'multipart-manifest': 'put'}) + + +class TestSloTempurl(Base): + env = TestSloTempurlEnv + set_up = False + + def setUp(self): + super(TestSloTempurl, self).setUp() + if self.env.enabled is False: + raise SkipTest("TempURL and SLO not both enabled") + elif self.env.enabled is not True: + # just some sanity checking + raise Exception( + "Expected enabled to be True/False, got %r" % + (self.env.enabled,)) + + def tempurl_sig(self, method, expires, path, key): + return hmac.new( + key, + '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), + hashlib.sha1).hexdigest() + + def test_GET(self): + expires = int(time.time()) + 86400 + sig = self.tempurl_sig( + 'GET', expires, self.env.conn.make_path(self.env.manifest.path), + self.env.tempurl_key) + parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} + + contents = self.env.manifest.read( + parms=parms, + cfg={'no_auth_token': True}) + self.assertEqual(len(contents), 2 * 1024 * 1024) + + # GET tempurls also allow HEAD requests + self.assertTrue(self.env.manifest.info( + parms=parms, cfg={'no_auth_token': True})) + + +class TestSloTempurlUTF8(Base2, TestSloTempurl): + set_up = False diff --git a/test/functional/tests.py b/test/functional/tests.py index fddbd0ec22..d335d1d35e 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -17,9 +17,6 @@ from datetime import datetime import email.parser import hashlib -import hmac -import itertools -import json import locale import random import six @@ -33,7 +30,7 @@ from unittest2 import SkipTest from swift.common.http import is_success, is_client_error from test.functional import normalized_urls, load_constraint, cluster_info -from test.functional import check_response, retry, requires_acls +from test.functional import check_response, retry import test.functional as tf from test.functional.swift_test_client import Account, Connection, File, \ ResponseError @@ -2506,374 +2503,6 @@ class TestFileUTF8(Base2, TestFile): set_up = False -class TestDloEnv(object): - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - - config2 = tf.config.copy() - config2['username'] = tf.config['username3'] - config2['password'] = tf.config['password3'] - cls.conn2 = Connection(config2) - cls.conn2.authenticate() - - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - - cls.container = cls.account.container(Utils.create_name()) - cls.container2 = cls.account.container(Utils.create_name()) - - for cont in (cls.container, cls.container2): - if not cont.create(): - raise ResponseError(cls.conn.response) - - # avoid getting a prefix that stops halfway through an encoded - # character - prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8") - cls.segment_prefix = prefix - - for letter in ('a', 'b', 'c', 'd', 'e'): - file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter)) - file_item.write(letter * 10) - - file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter)) - file_item.write(letter.upper() * 10) - - for letter in ('f', 'g', 'h', 'i', 'j'): - file_item = cls.container2.file("%s/seg_lower%s" % - (prefix, letter)) - file_item.write(letter * 10) - - man1 = cls.container.file("man1") - man1.write('man1-contents', - hdrs={"X-Object-Manifest": "%s/%s/seg_lower" % - (cls.container.name, prefix)}) - - man2 = cls.container.file("man2") - man2.write('man2-contents', - hdrs={"X-Object-Manifest": "%s/%s/seg_upper" % - (cls.container.name, prefix)}) - - manall = cls.container.file("manall") - manall.write('manall-contents', - hdrs={"X-Object-Manifest": "%s/%s/seg" % - (cls.container.name, prefix)}) - - mancont2 = cls.container.file("mancont2") - mancont2.write( - 'mancont2-contents', - hdrs={"X-Object-Manifest": "%s/%s/seg_lower" % - (cls.container2.name, prefix)}) - - -class TestDlo(Base): - env = TestDloEnv - set_up = False - - def test_get_manifest(self): - file_item = self.env.container.file('man1') - file_contents = file_item.read() - self.assertEqual( - file_contents, - "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") - - file_item = self.env.container.file('man2') - file_contents = file_item.read() - self.assertEqual( - file_contents, - "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE") - - file_item = self.env.container.file('manall') - file_contents = file_item.read() - self.assertEqual( - file_contents, - ("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" + - "AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")) - - def test_get_manifest_document_itself(self): - file_item = self.env.container.file('man1') - file_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual(file_contents, "man1-contents") - self.assertEqual(file_item.info()['x_object_manifest'], - "%s/%s/seg_lower" % - (self.env.container.name, self.env.segment_prefix)) - - def test_get_range(self): - file_item = self.env.container.file('man1') - file_contents = file_item.read(size=25, offset=8) - self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd") - - file_contents = file_item.read(size=1, offset=47) - self.assertEqual(file_contents, "e") - - def test_get_range_out_of_range(self): - file_item = self.env.container.file('man1') - - self.assertRaises(ResponseError, file_item.read, size=7, offset=50) - self.assert_status(416) - - def test_copy(self): - # Adding a new segment, copying the manifest, and then deleting the - # segment proves that the new object is really the concatenated - # segments and not just a manifest. - f_segment = self.env.container.file("%s/seg_lowerf" % - (self.env.segment_prefix)) - f_segment.write('ffffffffff') - try: - man1_item = self.env.container.file('man1') - man1_item.copy(self.env.container.name, "copied-man1") - finally: - # try not to leave this around for other tests to stumble over - f_segment.delete() - - file_item = self.env.container.file('copied-man1') - file_contents = file_item.read() - self.assertEqual( - file_contents, - "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") - # The copied object must not have X-Object-Manifest - self.assertNotIn("x_object_manifest", file_item.info()) - - def test_copy_account(self): - # dlo use same account and same container only - acct = self.env.conn.account_name - # Adding a new segment, copying the manifest, and then deleting the - # segment proves that the new object is really the concatenated - # segments and not just a manifest. - f_segment = self.env.container.file("%s/seg_lowerf" % - (self.env.segment_prefix)) - f_segment.write('ffffffffff') - try: - man1_item = self.env.container.file('man1') - man1_item.copy_account(acct, - self.env.container.name, - "copied-man1") - finally: - # try not to leave this around for other tests to stumble over - f_segment.delete() - - file_item = self.env.container.file('copied-man1') - file_contents = file_item.read() - self.assertEqual( - file_contents, - "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff") - # The copied object must not have X-Object-Manifest - self.assertNotIn("x_object_manifest", file_item.info()) - - def test_copy_manifest(self): - # Copying the manifest with multipart-manifest=get query string - # should result in another manifest - try: - man1_item = self.env.container.file('man1') - man1_item.copy(self.env.container.name, "copied-man1", - parms={'multipart-manifest': 'get'}) - - copied = self.env.container.file("copied-man1") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - self.assertEqual(copied_contents, "man1-contents") - - copied_contents = copied.read() - self.assertEqual( - copied_contents, - "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee") - self.assertEqual(man1_item.info()['x_object_manifest'], - copied.info()['x_object_manifest']) - finally: - # try not to leave this around for other tests to stumble over - self.env.container.file("copied-man1").delete() - - def test_dlo_if_match_get(self): - manifest = self.env.container.file("man1") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.read, - hdrs={'If-Match': 'not-%s' % etag}) - self.assert_status(412) - - manifest.read(hdrs={'If-Match': etag}) - self.assert_status(200) - - def test_dlo_if_none_match_get(self): - manifest = self.env.container.file("man1") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.read, - hdrs={'If-None-Match': etag}) - self.assert_status(304) - - manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) - self.assert_status(200) - - def test_dlo_if_match_head(self): - manifest = self.env.container.file("man1") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.info, - hdrs={'If-Match': 'not-%s' % etag}) - self.assert_status(412) - - manifest.info(hdrs={'If-Match': etag}) - self.assert_status(200) - - def test_dlo_if_none_match_head(self): - manifest = self.env.container.file("man1") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.info, - hdrs={'If-None-Match': etag}) - self.assert_status(304) - - manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) - self.assert_status(200) - - def test_dlo_referer_on_segment_container(self): - # First the account2 (test3) should fail - headers = {'X-Auth-Token': self.env.conn2.storage_token, - 'Referer': 'http://blah.example.com'} - dlo_file = self.env.container.file("mancont2") - self.assertRaises(ResponseError, dlo_file.read, - hdrs=headers) - self.assert_status(403) - - # Now set the referer on the dlo container only - referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} - self.env.container.update_metadata(referer_metadata) - - self.assertRaises(ResponseError, dlo_file.read, - hdrs=headers) - self.assert_status(403) - - # Finally set the referer on the segment container - self.env.container2.update_metadata(referer_metadata) - - contents = dlo_file.read(hdrs=headers) - self.assertEqual( - contents, - "ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj") - - def test_dlo_post_with_manifest_header(self): - # verify that performing a POST to a DLO manifest - # preserves the fact that it is a manifest file. - # verify that the x-object-manifest header may be updated. - - # create a new manifest for this test to avoid test coupling. - x_o_m = self.env.container.file('man1').info()['x_object_manifest'] - file_item = self.env.container.file(Utils.create_name()) - file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m}) - - # sanity checks - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('manifest-contents', manifest_contents) - expected_contents = ''.join([(c * 10) for c in 'abcde']) - contents = file_item.read(parms={}) - self.assertEqual(expected_contents, contents) - - # POST a modified x-object-manifest value - new_x_o_m = x_o_m.rstrip('lower') + 'upper' - file_item.post({'x-object-meta-foo': 'bar', - 'x-object-manifest': new_x_o_m}) - - # verify that x-object-manifest was updated - file_item.info() - resp_headers = file_item.conn.response.getheaders() - self.assertIn(('x-object-manifest', new_x_o_m), resp_headers) - self.assertIn(('x-object-meta-foo', 'bar'), resp_headers) - - # verify that manifest content was not changed - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('manifest-contents', manifest_contents) - - # verify that updated manifest points to new content - expected_contents = ''.join([(c * 10) for c in 'ABCDE']) - contents = file_item.read(parms={}) - self.assertEqual(expected_contents, contents) - - # Now revert the manifest to point to original segments, including a - # multipart-manifest=get param just to check that has no effect - file_item.post({'x-object-manifest': x_o_m}, - parms={'multipart-manifest': 'get'}) - - # verify that x-object-manifest was reverted - info = file_item.info() - self.assertIn('x_object_manifest', info) - self.assertEqual(x_o_m, info['x_object_manifest']) - - # verify that manifest content was not changed - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('manifest-contents', manifest_contents) - - # verify that updated manifest points new content - expected_contents = ''.join([(c * 10) for c in 'abcde']) - contents = file_item.read(parms={}) - self.assertEqual(expected_contents, contents) - - def test_dlo_post_without_manifest_header(self): - # verify that a POST to a DLO manifest object with no - # x-object-manifest header will cause the existing x-object-manifest - # header to be lost - - # create a new manifest for this test to avoid test coupling. - x_o_m = self.env.container.file('man1').info()['x_object_manifest'] - file_item = self.env.container.file(Utils.create_name()) - file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m}) - - # sanity checks - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('manifest-contents', manifest_contents) - expected_contents = ''.join([(c * 10) for c in 'abcde']) - contents = file_item.read(parms={}) - self.assertEqual(expected_contents, contents) - - # POST with no x-object-manifest header - file_item.post({}) - - # verify that existing x-object-manifest was removed - info = file_item.info() - self.assertNotIn('x_object_manifest', info) - - # verify that object content was not changed - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('manifest-contents', manifest_contents) - - # verify that object is no longer a manifest - contents = file_item.read(parms={}) - self.assertEqual('manifest-contents', contents) - - def test_dlo_post_with_manifest_regular_object(self): - # verify that performing a POST to a regular object - # with a manifest header will create a DLO. - - # Put a regular object - file_item = self.env.container.file(Utils.create_name()) - file_item.write('file contents', hdrs={}) - - # sanity checks - file_contents = file_item.read(parms={}) - self.assertEqual('file contents', file_contents) - - # get the path associated with man1 - x_o_m = self.env.container.file('man1').info()['x_object_manifest'] - - # POST a x-object-manifest value to the regular object - file_item.post({'x-object-manifest': x_o_m}) - - # verify that the file is now a manifest - manifest_contents = file_item.read(parms={'multipart-manifest': 'get'}) - self.assertEqual('file contents', manifest_contents) - expected_contents = ''.join([(c * 10) for c in 'abcde']) - contents = file_item.read(parms={}) - self.assertEqual(expected_contents, contents) - file_item.info() - resp_headers = file_item.conn.response.getheaders() - self.assertIn(('x-object-manifest', x_o_m), resp_headers) - - -class TestDloUTF8(Base2, TestDlo): - set_up = False - - class TestFileComparisonEnv(object): @classmethod def setUp(cls): @@ -3031,1544 +2660,6 @@ class TestFileComparisonUTF8(Base2, TestFileComparison): set_up = False -class TestSloEnv(object): - slo_enabled = None # tri-state: None initially, then True/False - - @classmethod - def create_segments(cls, container): - seg_info = {} - for letter, size in (('a', 1024 * 1024), - ('b', 1024 * 1024), - ('c', 1024 * 1024), - ('d', 1024 * 1024), - ('e', 1)): - seg_name = "seg_%s" % letter - file_item = container.file(seg_name) - file_item.write(letter * size) - seg_info[seg_name] = { - 'size_bytes': size, - 'etag': file_item.md5, - 'path': '/%s/%s' % (container.name, seg_name)} - return seg_info - - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - config2 = deepcopy(tf.config) - config2['account'] = tf.config['account2'] - config2['username'] = tf.config['username2'] - config2['password'] = tf.config['password2'] - cls.conn2 = Connection(config2) - cls.conn2.authenticate() - cls.account2 = cls.conn2.get_account() - cls.account2.delete_containers() - config3 = tf.config.copy() - config3['username'] = tf.config['username3'] - config3['password'] = tf.config['password3'] - cls.conn3 = Connection(config3) - cls.conn3.authenticate() - - if cls.slo_enabled is None: - cls.slo_enabled = 'slo' in cluster_info - if not cls.slo_enabled: - return - - cls.account = Account(cls.conn, tf.config.get('account', - tf.config['username'])) - cls.account.delete_containers() - - cls.container = cls.account.container(Utils.create_name()) - cls.container2 = cls.account.container(Utils.create_name()) - - for cont in (cls.container, cls.container2): - if not cont.create(): - raise ResponseError(cls.conn.response) - - cls.seg_info = seg_info = cls.create_segments(cls.container) - - file_item = cls.container.file("manifest-abcde") - file_item.write( - json.dumps([seg_info['seg_a'], seg_info['seg_b'], - seg_info['seg_c'], seg_info['seg_d'], - seg_info['seg_e']]), - parms={'multipart-manifest': 'put'}) - - # Put the same manifest in the container2 - file_item = cls.container2.file("manifest-abcde") - file_item.write( - json.dumps([seg_info['seg_a'], seg_info['seg_b'], - seg_info['seg_c'], seg_info['seg_d'], - seg_info['seg_e']]), - parms={'multipart-manifest': 'put'}) - - file_item = cls.container.file('manifest-cd') - cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']]) - file_item.write(cd_json, parms={'multipart-manifest': 'put'}) - cd_etag = hashlib.md5(seg_info['seg_c']['etag'] + - seg_info['seg_d']['etag']).hexdigest() - - file_item = cls.container.file("manifest-bcd-submanifest") - file_item.write( - json.dumps([seg_info['seg_b'], - {'etag': cd_etag, - 'size_bytes': (seg_info['seg_c']['size_bytes'] + - seg_info['seg_d']['size_bytes']), - 'path': '/%s/%s' % (cls.container.name, - 'manifest-cd')}]), - parms={'multipart-manifest': 'put'}) - bcd_submanifest_etag = hashlib.md5( - seg_info['seg_b']['etag'] + cd_etag).hexdigest() - - file_item = cls.container.file("manifest-abcde-submanifest") - file_item.write( - json.dumps([ - seg_info['seg_a'], - {'etag': bcd_submanifest_etag, - 'size_bytes': (seg_info['seg_b']['size_bytes'] + - seg_info['seg_c']['size_bytes'] + - seg_info['seg_d']['size_bytes']), - 'path': '/%s/%s' % (cls.container.name, - 'manifest-bcd-submanifest')}, - seg_info['seg_e']]), - parms={'multipart-manifest': 'put'}) - abcde_submanifest_etag = hashlib.md5( - seg_info['seg_a']['etag'] + bcd_submanifest_etag + - seg_info['seg_e']['etag']).hexdigest() - abcde_submanifest_size = (seg_info['seg_a']['size_bytes'] + - seg_info['seg_b']['size_bytes'] + - seg_info['seg_c']['size_bytes'] + - seg_info['seg_d']['size_bytes'] + - seg_info['seg_e']['size_bytes']) - - file_item = cls.container.file("ranged-manifest") - file_item.write( - json.dumps([ - {'etag': abcde_submanifest_etag, - 'size_bytes': abcde_submanifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'manifest-abcde-submanifest'), - 'range': '-1048578'}, # 'c' + ('d' * 2**20) + 'e' - {'etag': abcde_submanifest_etag, - 'size_bytes': abcde_submanifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'manifest-abcde-submanifest'), - 'range': '524288-1572863'}, # 'a' * 2**19 + 'b' * 2**19 - {'etag': abcde_submanifest_etag, - 'size_bytes': abcde_submanifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'manifest-abcde-submanifest'), - 'range': '3145727-3145728'}]), # 'cd' - parms={'multipart-manifest': 'put'}) - ranged_manifest_etag = hashlib.md5( - abcde_submanifest_etag + ':3145727-4194304;' + - abcde_submanifest_etag + ':524288-1572863;' + - abcde_submanifest_etag + ':3145727-3145728;').hexdigest() - ranged_manifest_size = 2 * 1024 * 1024 + 4 - - file_item = cls.container.file("ranged-submanifest") - file_item.write( - json.dumps([ - seg_info['seg_c'], - {'etag': ranged_manifest_etag, - 'size_bytes': ranged_manifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'ranged-manifest')}, - {'etag': ranged_manifest_etag, - 'size_bytes': ranged_manifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'ranged-manifest'), - 'range': '524289-1572865'}, - {'etag': ranged_manifest_etag, - 'size_bytes': ranged_manifest_size, - 'path': '/%s/%s' % (cls.container.name, - 'ranged-manifest'), - 'range': '-3'}]), - parms={'multipart-manifest': 'put'}) - - file_item = cls.container.file("manifest-db") - file_item.write( - json.dumps([ - {'path': seg_info['seg_d']['path'], 'etag': None, - 'size_bytes': None}, - {'path': seg_info['seg_b']['path'], 'etag': None, - 'size_bytes': None}, - ]), parms={'multipart-manifest': 'put'}) - - file_item = cls.container.file("ranged-manifest-repeated-segment") - file_item.write( - json.dumps([ - {'path': seg_info['seg_a']['path'], 'etag': None, - 'size_bytes': None, 'range': '-1048578'}, - {'path': seg_info['seg_a']['path'], 'etag': None, - 'size_bytes': None}, - {'path': seg_info['seg_b']['path'], 'etag': None, - 'size_bytes': None, 'range': '-1048578'}, - ]), parms={'multipart-manifest': 'put'}) - - -class TestSlo(Base): - env = TestSloEnv - set_up = False - - def setUp(self): - super(TestSlo, self).setUp() - if self.env.slo_enabled is False: - raise SkipTest("SLO not enabled") - elif self.env.slo_enabled is not True: - # just some sanity checking - raise Exception( - "Expected slo_enabled to be True/False, got %r" % - (self.env.slo_enabled,)) - - def test_slo_get_simple_manifest(self): - file_item = self.env.container.file('manifest-abcde') - file_contents = file_item.read() - self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) - self.assertEqual('a', file_contents[0]) - self.assertEqual('a', file_contents[1024 * 1024 - 1]) - self.assertEqual('b', file_contents[1024 * 1024]) - self.assertEqual('d', file_contents[-2]) - self.assertEqual('e', file_contents[-1]) - - def test_slo_container_listing(self): - # the listing object size should equal the sum of the size of the - # segments, not the size of the manifest body - file_item = self.env.container.file(Utils.create_name()) - file_item.write( - json.dumps([self.env.seg_info['seg_a']]), - parms={'multipart-manifest': 'put'}) - # The container listing has the etag of the actual manifest object - # contents which we get using multipart-manifest=get. Arguably this - # should be the etag that we get when NOT using multipart-manifest=get, - # to be consistent with size and content-type. But here we at least - # verify that it remains consistent when the object is updated with a - # POST. - file_item.initialize(parms={'multipart-manifest': 'get'}) - expected_etag = file_item.etag - - listing = self.env.container.files(parms={'format': 'json'}) - for f_dict in listing: - if f_dict['name'] == file_item.name: - self.assertEqual(1024 * 1024, f_dict['bytes']) - self.assertEqual('application/octet-stream', - f_dict['content_type']) - self.assertEqual(expected_etag, f_dict['hash']) - break - else: - self.fail('Failed to find manifest file in container listing') - - # now POST updated content-type file - file_item.content_type = 'image/jpeg' - file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}) - file_item.initialize() - self.assertEqual('image/jpeg', file_item.content_type) # sanity - - # verify that the container listing is consistent with the file - listing = self.env.container.files(parms={'format': 'json'}) - for f_dict in listing: - if f_dict['name'] == file_item.name: - self.assertEqual(1024 * 1024, f_dict['bytes']) - self.assertEqual(file_item.content_type, - f_dict['content_type']) - self.assertEqual(expected_etag, f_dict['hash']) - break - else: - self.fail('Failed to find manifest file in container listing') - - # now POST with no change to content-type - file_item.sync_metadata({'X-Object-Meta-Test': 'blah'}, - cfg={'no_content_type': True}) - file_item.initialize() - self.assertEqual('image/jpeg', file_item.content_type) # sanity - - # verify that the container listing is consistent with the file - listing = self.env.container.files(parms={'format': 'json'}) - for f_dict in listing: - if f_dict['name'] == file_item.name: - self.assertEqual(1024 * 1024, f_dict['bytes']) - self.assertEqual(file_item.content_type, - f_dict['content_type']) - self.assertEqual(expected_etag, f_dict['hash']) - break - else: - self.fail('Failed to find manifest file in container listing') - - def test_slo_get_nested_manifest(self): - file_item = self.env.container.file('manifest-abcde-submanifest') - file_contents = file_item.read() - self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents)) - self.assertEqual('a', file_contents[0]) - self.assertEqual('a', file_contents[1024 * 1024 - 1]) - self.assertEqual('b', file_contents[1024 * 1024]) - self.assertEqual('d', file_contents[-2]) - self.assertEqual('e', file_contents[-1]) - - def test_slo_get_ranged_manifest(self): - file_item = self.env.container.file('ranged-manifest') - grouped_file_contents = [ - (char, sum(1 for _char in grp)) - for char, grp in itertools.groupby(file_item.read())] - self.assertEqual([ - ('c', 1), - ('d', 1024 * 1024), - ('e', 1), - ('a', 512 * 1024), - ('b', 512 * 1024), - ('c', 1), - ('d', 1)], grouped_file_contents) - - def test_slo_get_ranged_manifest_repeated_segment(self): - file_item = self.env.container.file('ranged-manifest-repeated-segment') - grouped_file_contents = [ - (char, sum(1 for _char in grp)) - for char, grp in itertools.groupby(file_item.read())] - self.assertEqual( - [('a', 2097152), ('b', 1048576)], - grouped_file_contents) - - def test_slo_get_ranged_submanifest(self): - file_item = self.env.container.file('ranged-submanifest') - grouped_file_contents = [ - (char, sum(1 for _char in grp)) - for char, grp in itertools.groupby(file_item.read())] - self.assertEqual([ - ('c', 1024 * 1024 + 1), - ('d', 1024 * 1024), - ('e', 1), - ('a', 512 * 1024), - ('b', 512 * 1024), - ('c', 1), - ('d', 512 * 1024 + 1), - ('e', 1), - ('a', 512 * 1024), - ('b', 1), - ('c', 1), - ('d', 1)], grouped_file_contents) - - def test_slo_ranged_get(self): - file_item = self.env.container.file('manifest-abcde') - file_contents = file_item.read(size=1024 * 1024 + 2, - offset=1024 * 1024 - 1) - self.assertEqual('a', file_contents[0]) - self.assertEqual('b', file_contents[1]) - self.assertEqual('b', file_contents[-2]) - self.assertEqual('c', file_contents[-1]) - - def test_slo_multi_ranged_get(self): - file_item = self.env.container.file('manifest-abcde') - file_contents = file_item.read( - hdrs={"Range": "bytes=1048571-1048580,2097147-2097156"}) - - # See testMultiRangeGets for explanation - parser = email.parser.FeedParser() - parser.feed("Content-Type: %s\r\n\r\n" % file_item.content_type) - parser.feed(file_contents) - - root_message = parser.close() - self.assertTrue(root_message.is_multipart()) # sanity check - - byteranges = root_message.get_payload() - self.assertEqual(len(byteranges), 2) - - self.assertEqual(byteranges[0]['Content-Type'], - "application/octet-stream") - self.assertEqual( - byteranges[0]['Content-Range'], "bytes 1048571-1048580/4194305") - self.assertEqual(byteranges[0].get_payload(), "aaaaabbbbb") - - self.assertEqual(byteranges[1]['Content-Type'], - "application/octet-stream") - self.assertEqual( - byteranges[1]['Content-Range'], "bytes 2097147-2097156/4194305") - self.assertEqual(byteranges[1].get_payload(), "bbbbbccccc") - - def test_slo_ranged_submanifest(self): - file_item = self.env.container.file('manifest-abcde-submanifest') - file_contents = file_item.read(size=1024 * 1024 + 2, - offset=1024 * 1024 * 2 - 1) - self.assertEqual('b', file_contents[0]) - self.assertEqual('c', file_contents[1]) - self.assertEqual('c', file_contents[-2]) - self.assertEqual('d', file_contents[-1]) - - def test_slo_etag_is_hash_of_etags(self): - expected_hash = hashlib.md5() - expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest()) - expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest()) - expected_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest()) - expected_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest()) - expected_hash.update(hashlib.md5('e').hexdigest()) - expected_etag = expected_hash.hexdigest() - - file_item = self.env.container.file('manifest-abcde') - self.assertEqual(expected_etag, file_item.info()['etag']) - - def test_slo_etag_is_hash_of_etags_submanifests(self): - - def hd(x): - return hashlib.md5(x).hexdigest() - - expected_etag = hd(hd('a' * 1024 * 1024) + - hd(hd('b' * 1024 * 1024) + - hd(hd('c' * 1024 * 1024) + - hd('d' * 1024 * 1024))) + - hd('e')) - - file_item = self.env.container.file('manifest-abcde-submanifest') - self.assertEqual(expected_etag, file_item.info()['etag']) - - def test_slo_etag_mismatch(self): - file_item = self.env.container.file("manifest-a-bad-etag") - try: - file_item.write( - json.dumps([{ - 'size_bytes': 1024 * 1024, - 'etag': 'not it', - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - except ResponseError as err: - self.assertEqual(400, err.status) - else: - self.fail("Expected ResponseError but didn't get it") - - def test_slo_size_mismatch(self): - file_item = self.env.container.file("manifest-a-bad-size") - try: - file_item.write( - json.dumps([{ - 'size_bytes': 1024 * 1024 - 1, - 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - except ResponseError as err: - self.assertEqual(400, err.status) - else: - self.fail("Expected ResponseError but didn't get it") - - def test_slo_unspecified_etag(self): - file_item = self.env.container.file("manifest-a-unspecified-etag") - file_item.write( - json.dumps([{ - 'size_bytes': 1024 * 1024, - 'etag': None, - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - self.assert_status(201) - - def test_slo_unspecified_size(self): - file_item = self.env.container.file("manifest-a-unspecified-size") - file_item.write( - json.dumps([{ - 'size_bytes': None, - 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - self.assert_status(201) - - def test_slo_missing_etag(self): - file_item = self.env.container.file("manifest-a-missing-etag") - try: - file_item.write( - json.dumps([{ - 'size_bytes': 1024 * 1024, - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - except ResponseError as err: - self.assertEqual(400, err.status) - else: - self.fail("Expected ResponseError but didn't get it") - - def test_slo_missing_size(self): - file_item = self.env.container.file("manifest-a-missing-size") - try: - file_item.write( - json.dumps([{ - 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]), - parms={'multipart-manifest': 'put'}) - except ResponseError as err: - self.assertEqual(400, err.status) - else: - self.fail("Expected ResponseError but didn't get it") - - def test_slo_overwrite_segment_with_manifest(self): - file_item = self.env.container.file("seg_b") - with self.assertRaises(ResponseError) as catcher: - file_item.write( - json.dumps([ - {'size_bytes': 1024 * 1024, - 'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}, - {'size_bytes': 1024 * 1024, - 'etag': hashlib.md5('b' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_b')}, - {'size_bytes': 1024 * 1024, - 'etag': hashlib.md5('c' * 1024 * 1024).hexdigest(), - 'path': '/%s/%s' % (self.env.container.name, 'seg_c')}]), - parms={'multipart-manifest': 'put'}) - self.assertEqual(400, catcher.exception.status) - - def test_slo_copy(self): - file_item = self.env.container.file("manifest-abcde") - file_item.copy(self.env.container.name, "copied-abcde") - - copied = self.env.container.file("copied-abcde") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) - - def test_slo_copy_account(self): - acct = self.env.conn.account_name - # same account copy - file_item = self.env.container.file("manifest-abcde") - file_item.copy_account(acct, self.env.container.name, "copied-abcde") - - copied = self.env.container.file("copied-abcde") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) - - # copy to different account - acct = self.env.conn2.account_name - dest_cont = self.env.account2.container(Utils.create_name()) - self.assertTrue(dest_cont.create(hdrs={ - 'X-Container-Write': self.env.conn.user_acl - })) - file_item = self.env.container.file("manifest-abcde") - file_item.copy_account(acct, dest_cont, "copied-abcde") - - copied = dest_cont.file("copied-abcde") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents)) - - def test_slo_copy_the_manifest(self): - source = self.env.container.file("manifest-abcde") - source_contents = source.read(parms={'multipart-manifest': 'get'}) - source_json = json.loads(source_contents) - source.initialize() - self.assertEqual('application/octet-stream', source.content_type) - source.initialize(parms={'multipart-manifest': 'get'}) - source_hash = hashlib.md5() - source_hash.update(source_contents) - self.assertEqual(source_hash.hexdigest(), source.etag) - - self.assertTrue(source.copy(self.env.container.name, - "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'})) - - copied = self.env.container.file("copied-abcde-manifest-only") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - try: - copied_json = json.loads(copied_contents) - except ValueError: - self.fail("COPY didn't copy the manifest (invalid json on GET)") - self.assertEqual(source_json, copied_json) - copied.initialize() - self.assertEqual('application/octet-stream', copied.content_type) - copied.initialize(parms={'multipart-manifest': 'get'}) - copied_hash = hashlib.md5() - copied_hash.update(copied_contents) - self.assertEqual(copied_hash.hexdigest(), copied.etag) - - # verify the listing metadata - listing = self.env.container.files(parms={'format': 'json'}) - names = {} - for f_dict in listing: - if f_dict['name'] in ('manifest-abcde', - 'copied-abcde-manifest-only'): - names[f_dict['name']] = f_dict - - self.assertIn('manifest-abcde', names) - actual = names['manifest-abcde'] - self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) - self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(source.etag, actual['hash']) - - self.assertIn('copied-abcde-manifest-only', names) - actual = names['copied-abcde-manifest-only'] - self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) - self.assertEqual('application/octet-stream', actual['content_type']) - self.assertEqual(copied.etag, actual['hash']) - - def test_slo_copy_the_manifest_updating_metadata(self): - source = self.env.container.file("manifest-abcde") - source.content_type = 'application/octet-stream' - source.sync_metadata({'test': 'original'}) - source_contents = source.read(parms={'multipart-manifest': 'get'}) - source_json = json.loads(source_contents) - source.initialize() - self.assertEqual('application/octet-stream', source.content_type) - source.initialize(parms={'multipart-manifest': 'get'}) - source_hash = hashlib.md5() - source_hash.update(source_contents) - self.assertEqual(source_hash.hexdigest(), source.etag) - self.assertEqual(source.metadata['test'], 'original') - - self.assertTrue( - source.copy(self.env.container.name, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}, - hdrs={'Content-Type': 'image/jpeg', - 'X-Object-Meta-Test': 'updated'})) - - copied = self.env.container.file("copied-abcde-manifest-only") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - try: - copied_json = json.loads(copied_contents) - except ValueError: - self.fail("COPY didn't copy the manifest (invalid json on GET)") - self.assertEqual(source_json, copied_json) - copied.initialize() - self.assertEqual('image/jpeg', copied.content_type) - copied.initialize(parms={'multipart-manifest': 'get'}) - copied_hash = hashlib.md5() - copied_hash.update(copied_contents) - self.assertEqual(copied_hash.hexdigest(), copied.etag) - self.assertEqual(copied.metadata['test'], 'updated') - - # verify the listing metadata - listing = self.env.container.files(parms={'format': 'json'}) - names = {} - for f_dict in listing: - if f_dict['name'] in ('manifest-abcde', - 'copied-abcde-manifest-only'): - names[f_dict['name']] = f_dict - - self.assertIn('manifest-abcde', names) - actual = names['manifest-abcde'] - self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) - self.assertEqual('application/octet-stream', actual['content_type']) - # the container listing should have the etag of the manifest contents - self.assertEqual(source.etag, actual['hash']) - - self.assertIn('copied-abcde-manifest-only', names) - actual = names['copied-abcde-manifest-only'] - self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes']) - self.assertEqual('image/jpeg', actual['content_type']) - self.assertEqual(copied.etag, actual['hash']) - - def test_slo_copy_the_manifest_account(self): - acct = self.env.conn.account_name - # same account - file_item = self.env.container.file("manifest-abcde") - file_item.copy_account(acct, - self.env.container.name, - "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) - - copied = self.env.container.file("copied-abcde-manifest-only") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - try: - json.loads(copied_contents) - except ValueError: - self.fail("COPY didn't copy the manifest (invalid json on GET)") - - # different account - acct = self.env.conn2.account_name - dest_cont = self.env.account2.container(Utils.create_name()) - self.assertTrue(dest_cont.create(hdrs={ - 'X-Container-Write': self.env.conn.user_acl - })) - - # manifest copy will fail because there is no read access to segments - # in destination account - file_item.copy_account( - acct, dest_cont, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) - self.assertEqual(400, file_item.conn.response.status) - resp_body = file_item.conn.response.read() - self.assertEqual(5, resp_body.count('403 Forbidden'), - 'Unexpected response body %r' % resp_body) - - # create segments container in account2 with read access for account1 - segs_container = self.env.account2.container(self.env.container.name) - self.assertTrue(segs_container.create(hdrs={ - 'X-Container-Read': self.env.conn.user_acl - })) - - # manifest copy will still fail because there are no segments in - # destination account - file_item.copy_account( - acct, dest_cont, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'}) - self.assertEqual(400, file_item.conn.response.status) - resp_body = file_item.conn.response.read() - self.assertEqual(5, resp_body.count('404 Not Found'), - 'Unexpected response body %r' % resp_body) - - # create segments in account2 container with same name as in account1, - # manifest copy now succeeds - self.env.create_segments(segs_container) - - self.assertTrue(file_item.copy_account( - acct, dest_cont, "copied-abcde-manifest-only", - parms={'multipart-manifest': 'get'})) - - copied = dest_cont.file("copied-abcde-manifest-only") - copied_contents = copied.read(parms={'multipart-manifest': 'get'}) - try: - json.loads(copied_contents) - except ValueError: - self.fail("COPY didn't copy the manifest (invalid json on GET)") - - def _make_manifest(self): - file_item = self.env.container.file("manifest-post") - seg_info = self.env.seg_info - file_item.write( - json.dumps([seg_info['seg_a'], seg_info['seg_b'], - seg_info['seg_c'], seg_info['seg_d'], - seg_info['seg_e']]), - parms={'multipart-manifest': 'put'}) - return file_item - - def test_slo_post_the_manifest_metadata_update(self): - file_item = self._make_manifest() - # sanity check, check the object is an SLO manifest - file_item.info() - file_item.header_fields([('slo', 'x-static-large-object')]) - - # POST a user metadata (i.e. x-object-meta-post) - file_item.sync_metadata({'post': 'update'}) - - updated = self.env.container.file("manifest-post") - updated.info() - updated.header_fields([('user-meta', 'x-object-meta-post')]) # sanity - updated.header_fields([('slo', 'x-static-large-object')]) - updated_contents = updated.read(parms={'multipart-manifest': 'get'}) - try: - json.loads(updated_contents) - except ValueError: - self.fail("Unexpected content on GET, expected a json body") - - def test_slo_post_the_manifest_metadata_update_with_qs(self): - # multipart-manifest query should be ignored on post - for verb in ('put', 'get', 'delete'): - file_item = self._make_manifest() - # sanity check, check the object is an SLO manifest - file_item.info() - file_item.header_fields([('slo', 'x-static-large-object')]) - # POST a user metadata (i.e. x-object-meta-post) - file_item.sync_metadata(metadata={'post': 'update'}, - parms={'multipart-manifest': verb}) - updated = self.env.container.file("manifest-post") - updated.info() - updated.header_fields( - [('user-meta', 'x-object-meta-post')]) # sanity - updated.header_fields([('slo', 'x-static-large-object')]) - updated_contents = updated.read( - parms={'multipart-manifest': 'get'}) - try: - json.loads(updated_contents) - except ValueError: - self.fail( - "Unexpected content on GET, expected a json body") - - def test_slo_get_the_manifest(self): - manifest = self.env.container.file("manifest-abcde") - got_body = manifest.read(parms={'multipart-manifest': 'get'}) - - self.assertEqual('application/json; charset=utf-8', - manifest.content_type) - try: - json.loads(got_body) - except ValueError: - self.fail("GET with multipart-manifest=get got invalid json") - - def test_slo_get_the_manifest_with_details_from_server(self): - manifest = self.env.container.file("manifest-db") - got_body = manifest.read(parms={'multipart-manifest': 'get'}) - - self.assertEqual('application/json; charset=utf-8', - manifest.content_type) - try: - value = json.loads(got_body) - except ValueError: - self.fail("GET with multipart-manifest=get got invalid json") - - self.assertEqual(len(value), 2) - self.assertEqual(value[0]['bytes'], 1024 * 1024) - self.assertEqual(value[0]['hash'], - hashlib.md5('d' * 1024 * 1024).hexdigest()) - self.assertEqual(value[0]['name'], - '/%s/seg_d' % self.env.container.name.decode("utf-8")) - - self.assertEqual(value[1]['bytes'], 1024 * 1024) - self.assertEqual(value[1]['hash'], - hashlib.md5('b' * 1024 * 1024).hexdigest()) - self.assertEqual(value[1]['name'], - '/%s/seg_b' % self.env.container.name.decode("utf-8")) - - def test_slo_get_raw_the_manifest_with_details_from_server(self): - manifest = self.env.container.file("manifest-db") - got_body = manifest.read(parms={'multipart-manifest': 'get', - 'format': 'raw'}) - - # raw format should have the actual manifest object content-type - self.assertEqual('application/octet-stream', manifest.content_type) - try: - value = json.loads(got_body) - except ValueError: - msg = "GET with multipart-manifest=get&format=raw got invalid json" - self.fail(msg) - - self.assertEqual( - set(value[0].keys()), set(('size_bytes', 'etag', 'path'))) - self.assertEqual(len(value), 2) - self.assertEqual(value[0]['size_bytes'], 1024 * 1024) - self.assertEqual(value[0]['etag'], - hashlib.md5('d' * 1024 * 1024).hexdigest()) - self.assertEqual(value[0]['path'], - '/%s/seg_d' % self.env.container.name.decode("utf-8")) - self.assertEqual(value[1]['size_bytes'], 1024 * 1024) - self.assertEqual(value[1]['etag'], - hashlib.md5('b' * 1024 * 1024).hexdigest()) - self.assertEqual(value[1]['path'], - '/%s/seg_b' % self.env.container.name.decode("utf-8")) - - file_item = self.env.container.file("manifest-from-get-raw") - file_item.write(got_body, parms={'multipart-manifest': 'put'}) - - file_contents = file_item.read() - self.assertEqual(2 * 1024 * 1024, len(file_contents)) - - def test_slo_head_the_manifest(self): - manifest = self.env.container.file("manifest-abcde") - got_info = manifest.info(parms={'multipart-manifest': 'get'}) - - self.assertEqual('application/json; charset=utf-8', - got_info['content_type']) - - def test_slo_if_match_get(self): - manifest = self.env.container.file("manifest-abcde") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.read, - hdrs={'If-Match': 'not-%s' % etag}) - self.assert_status(412) - - manifest.read(hdrs={'If-Match': etag}) - self.assert_status(200) - - def test_slo_if_none_match_put(self): - file_item = self.env.container.file("manifest-if-none-match") - manifest = json.dumps([{ - 'size_bytes': 1024 * 1024, - 'etag': None, - 'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]) - - self.assertRaises(ResponseError, file_item.write, manifest, - parms={'multipart-manifest': 'put'}, - hdrs={'If-None-Match': '"not-star"'}) - self.assert_status(400) - - file_item.write(manifest, parms={'multipart-manifest': 'put'}, - hdrs={'If-None-Match': '*'}) - self.assert_status(201) - - self.assertRaises(ResponseError, file_item.write, manifest, - parms={'multipart-manifest': 'put'}, - hdrs={'If-None-Match': '*'}) - self.assert_status(412) - - def test_slo_if_none_match_get(self): - manifest = self.env.container.file("manifest-abcde") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.read, - hdrs={'If-None-Match': etag}) - self.assert_status(304) - - manifest.read(hdrs={'If-None-Match': "not-%s" % etag}) - self.assert_status(200) - - def test_slo_if_match_head(self): - manifest = self.env.container.file("manifest-abcde") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.info, - hdrs={'If-Match': 'not-%s' % etag}) - self.assert_status(412) - - manifest.info(hdrs={'If-Match': etag}) - self.assert_status(200) - - def test_slo_if_none_match_head(self): - manifest = self.env.container.file("manifest-abcde") - etag = manifest.info()['etag'] - - self.assertRaises(ResponseError, manifest.info, - hdrs={'If-None-Match': etag}) - self.assert_status(304) - - manifest.info(hdrs={'If-None-Match': "not-%s" % etag}) - self.assert_status(200) - - def test_slo_referer_on_segment_container(self): - # First the account2 (test3) should fail - headers = {'X-Auth-Token': self.env.conn3.storage_token, - 'Referer': 'http://blah.example.com'} - slo_file = self.env.container2.file('manifest-abcde') - self.assertRaises(ResponseError, slo_file.read, - hdrs=headers) - self.assert_status(403) - - # Now set the referer on the slo container only - referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} - self.env.container2.update_metadata(referer_metadata) - - self.assertRaises(ResponseError, slo_file.read, - hdrs=headers) - self.assert_status(409) - - # Finally set the referer on the segment container - self.env.container.update_metadata(referer_metadata) - contents = slo_file.read(hdrs=headers) - self.assertEqual(4 * 1024 * 1024 + 1, len(contents)) - self.assertEqual('a', contents[0]) - self.assertEqual('a', contents[1024 * 1024 - 1]) - self.assertEqual('b', contents[1024 * 1024]) - self.assertEqual('d', contents[-2]) - self.assertEqual('e', contents[-1]) - - -class TestSloUTF8(Base2, TestSlo): - set_up = False - - -class TestTempurlEnv(object): - tempurl_enabled = None # tri-state: None initially, then True/False - - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - - if cls.tempurl_enabled is None: - cls.tempurl_enabled = 'tempurl' in cluster_info - if not cls.tempurl_enabled: - return - - cls.tempurl_key = Utils.create_name() - cls.tempurl_key2 = Utils.create_name() - - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() - cls.account.update_metadata({ - 'temp-url-key': cls.tempurl_key, - 'temp-url-key-2': cls.tempurl_key2 - }) - - cls.container = cls.account.container(Utils.create_name()) - if not cls.container.create(): - raise ResponseError(cls.conn.response) - - cls.obj = cls.container.file(Utils.create_name()) - cls.obj.write("obj contents") - cls.other_obj = cls.container.file(Utils.create_name()) - cls.other_obj.write("other obj contents") - - -class TestTempurl(Base): - env = TestTempurlEnv - set_up = False - - def setUp(self): - super(TestTempurl, self).setUp() - if self.env.tempurl_enabled is False: - raise SkipTest("TempURL not enabled") - elif self.env.tempurl_enabled is not True: - # just some sanity checking - raise Exception( - "Expected tempurl_enabled to be True/False, got %r" % - (self.env.tempurl_enabled,)) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key) - self.obj_tempurl_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - def tempurl_sig(self, method, expires, path, key): - return hmac.new( - key, - '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), - hashlib.sha1).hexdigest() - - def test_GET(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - # GET tempurls also allow HEAD requests - self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True})) - - def test_GET_with_key_2(self): - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key2) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - def test_GET_DLO_inside_container(self): - seg1 = self.env.container.file( - "get-dlo-inside-seg1" + Utils.create_name()) - seg2 = self.env.container.file( - "get-dlo-inside-seg2" + Utils.create_name()) - seg1.write("one fish two fish ") - seg2.write("red fish blue fish") - - manifest = self.env.container.file("manifest" + Utils.create_name()) - manifest.write( - '', - hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % - (self.env.container.name,)}) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(manifest.path), - self.env.tempurl_key) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) - self.assertEqual(contents, "one fish two fish red fish blue fish") - - def test_GET_DLO_outside_container(self): - seg1 = self.env.container.file( - "get-dlo-outside-seg1" + Utils.create_name()) - seg2 = self.env.container.file( - "get-dlo-outside-seg2" + Utils.create_name()) - seg1.write("one fish two fish ") - seg2.write("red fish blue fish") - - container2 = self.env.account.container(Utils.create_name()) - container2.create() - - manifest = container2.file("manifest" + Utils.create_name()) - manifest.write( - '', - hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % - (self.env.container.name,)}) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(manifest.path), - self.env.tempurl_key) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - # cross container tempurl works fine for account tempurl key - contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) - self.assertEqual(contents, "one fish two fish red fish blue fish") - self.assert_status([200]) - - def test_PUT(self): - new_obj = self.env.container.file(Utils.create_name()) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'PUT', expires, self.env.conn.make_path(new_obj.path), - self.env.tempurl_key) - put_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - new_obj.write('new obj contents', - parms=put_parms, cfg={'no_auth_token': True}) - self.assertEqual(new_obj.read(), "new obj contents") - - # PUT tempurls also allow HEAD requests - self.assertTrue(new_obj.info(parms=put_parms, - cfg={'no_auth_token': True})) - - def test_PUT_manifest_access(self): - new_obj = self.env.container.file(Utils.create_name()) - - # give out a signature which allows a PUT to new_obj - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'PUT', expires, self.env.conn.make_path(new_obj.path), - self.env.tempurl_key) - put_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - # try to create manifest pointing to some random container - try: - new_obj.write('', { - 'x-object-manifest': '%s/foo' % 'some_random_container' - }, parms=put_parms, cfg={'no_auth_token': True}) - except ResponseError as e: - self.assertEqual(e.status, 400) - else: - self.fail('request did not error') - - # create some other container - other_container = self.env.account.container(Utils.create_name()) - if not other_container.create(): - raise ResponseError(self.conn.response) - - # try to create manifest pointing to new container - try: - new_obj.write('', { - 'x-object-manifest': '%s/foo' % other_container - }, parms=put_parms, cfg={'no_auth_token': True}) - except ResponseError as e: - self.assertEqual(e.status, 400) - else: - self.fail('request did not error') - - # try again using a tempurl POST to an already created object - new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True}) - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'POST', expires, self.env.conn.make_path(new_obj.path), - self.env.tempurl_key) - post_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - try: - new_obj.post({'x-object-manifest': '%s/foo' % other_container}, - parms=post_parms, cfg={'no_auth_token': True}) - except ResponseError as e: - self.assertEqual(e.status, 400) - else: - self.fail('request did not error') - - def test_HEAD(self): - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key) - head_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - self.assertTrue(self.env.obj.info(parms=head_parms, - cfg={'no_auth_token': True})) - # HEAD tempurls don't allow PUT or GET requests, despite the fact that - # PUT and GET tempurls both allow HEAD requests - self.assertRaises(ResponseError, self.env.other_obj.read, - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - self.assertRaises(ResponseError, self.env.other_obj.write, - 'new contents', - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - def test_different_object(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - self.assertRaises(ResponseError, self.env.other_obj.read, - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - def test_changing_sig(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - parms = self.obj_tempurl_parms.copy() - if parms['temp_url_sig'][0] == 'a': - parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] - else: - parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] - - self.assertRaises(ResponseError, self.env.obj.read, - cfg={'no_auth_token': True}, - parms=parms) - self.assert_status([401]) - - def test_changing_expires(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - parms = self.obj_tempurl_parms.copy() - if parms['temp_url_expires'][-1] == '0': - parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' - else: - parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' - - self.assertRaises(ResponseError, self.env.obj.read, - cfg={'no_auth_token': True}, - parms=parms) - self.assert_status([401]) - - -class TestTempurlUTF8(Base2, TestTempurl): - set_up = False - - -class TestContainerTempurlEnv(object): - tempurl_enabled = None # tri-state: None initially, then True/False - - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - - if cls.tempurl_enabled is None: - cls.tempurl_enabled = 'tempurl' in cluster_info - if not cls.tempurl_enabled: - return - - cls.tempurl_key = Utils.create_name() - cls.tempurl_key2 = Utils.create_name() - - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() - - # creating another account and connection - # for ACL tests - config2 = deepcopy(tf.config) - config2['account'] = tf.config['account2'] - config2['username'] = tf.config['username2'] - config2['password'] = tf.config['password2'] - cls.conn2 = Connection(config2) - cls.conn2.authenticate() - cls.account2 = Account( - cls.conn2, config2.get('account', config2['username'])) - cls.account2 = cls.conn2.get_account() - - cls.container = cls.account.container(Utils.create_name()) - if not cls.container.create({ - 'x-container-meta-temp-url-key': cls.tempurl_key, - 'x-container-meta-temp-url-key-2': cls.tempurl_key2, - 'x-container-read': cls.account2.name}): - raise ResponseError(cls.conn.response) - - cls.obj = cls.container.file(Utils.create_name()) - cls.obj.write("obj contents") - cls.other_obj = cls.container.file(Utils.create_name()) - cls.other_obj.write("other obj contents") - - -class TestContainerTempurl(Base): - env = TestContainerTempurlEnv - set_up = False - - def setUp(self): - super(TestContainerTempurl, self).setUp() - if self.env.tempurl_enabled is False: - raise SkipTest("TempURL not enabled") - elif self.env.tempurl_enabled is not True: - # just some sanity checking - raise Exception( - "Expected tempurl_enabled to be True/False, got %r" % - (self.env.tempurl_enabled,)) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key) - self.obj_tempurl_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - def tempurl_sig(self, method, expires, path, key): - return hmac.new( - key, - '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), - hashlib.sha1).hexdigest() - - def test_GET(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - # GET tempurls also allow HEAD requests - self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True})) - - def test_GET_with_key_2(self): - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key2) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - def test_PUT(self): - new_obj = self.env.container.file(Utils.create_name()) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'PUT', expires, self.env.conn.make_path(new_obj.path), - self.env.tempurl_key) - put_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - new_obj.write('new obj contents', - parms=put_parms, cfg={'no_auth_token': True}) - self.assertEqual(new_obj.read(), "new obj contents") - - # PUT tempurls also allow HEAD requests - self.assertTrue(new_obj.info(parms=put_parms, - cfg={'no_auth_token': True})) - - def test_HEAD(self): - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'HEAD', expires, self.env.conn.make_path(self.env.obj.path), - self.env.tempurl_key) - head_parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - self.assertTrue(self.env.obj.info(parms=head_parms, - cfg={'no_auth_token': True})) - # HEAD tempurls don't allow PUT or GET requests, despite the fact that - # PUT and GET tempurls both allow HEAD requests - self.assertRaises(ResponseError, self.env.other_obj.read, - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - self.assertRaises(ResponseError, self.env.other_obj.write, - 'new contents', - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - def test_different_object(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - self.assertRaises(ResponseError, self.env.other_obj.read, - cfg={'no_auth_token': True}, - parms=self.obj_tempurl_parms) - self.assert_status([401]) - - def test_changing_sig(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - parms = self.obj_tempurl_parms.copy() - if parms['temp_url_sig'][0] == 'a': - parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:] - else: - parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:] - - self.assertRaises(ResponseError, self.env.obj.read, - cfg={'no_auth_token': True}, - parms=parms) - self.assert_status([401]) - - def test_changing_expires(self): - contents = self.env.obj.read( - parms=self.obj_tempurl_parms, - cfg={'no_auth_token': True}) - self.assertEqual(contents, "obj contents") - - parms = self.obj_tempurl_parms.copy() - if parms['temp_url_expires'][-1] == '0': - parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1' - else: - parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0' - - self.assertRaises(ResponseError, self.env.obj.read, - cfg={'no_auth_token': True}, - parms=parms) - self.assert_status([401]) - - @requires_acls - def test_tempurl_keys_visible_to_account_owner(self): - if not tf.cluster_info.get('tempauth'): - raise SkipTest('TEMP AUTH SPECIFIC TEST') - metadata = self.env.container.info() - self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key) - self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2) - - @requires_acls - def test_tempurl_keys_hidden_from_acl_readonly(self): - if not tf.cluster_info.get('tempauth'): - raise SkipTest('TEMP AUTH SPECIFIC TEST') - original_token = self.env.container.conn.storage_token - self.env.container.conn.storage_token = self.env.conn2.storage_token - metadata = self.env.container.info() - self.env.container.conn.storage_token = original_token - - self.assertNotIn( - 'tempurl_key', metadata, - 'Container TempURL key found, should not be visible ' - 'to readonly ACLs') - self.assertNotIn( - 'tempurl_key2', metadata, - 'Container TempURL key-2 found, should not be visible ' - 'to readonly ACLs') - - def test_GET_DLO_inside_container(self): - seg1 = self.env.container.file( - "get-dlo-inside-seg1" + Utils.create_name()) - seg2 = self.env.container.file( - "get-dlo-inside-seg2" + Utils.create_name()) - seg1.write("one fish two fish ") - seg2.write("red fish blue fish") - - manifest = self.env.container.file("manifest" + Utils.create_name()) - manifest.write( - '', - hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" % - (self.env.container.name,)}) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(manifest.path), - self.env.tempurl_key) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - contents = manifest.read(parms=parms, cfg={'no_auth_token': True}) - self.assertEqual(contents, "one fish two fish red fish blue fish") - - def test_GET_DLO_outside_container(self): - container2 = self.env.account.container(Utils.create_name()) - container2.create() - seg1 = container2.file( - "get-dlo-outside-seg1" + Utils.create_name()) - seg2 = container2.file( - "get-dlo-outside-seg2" + Utils.create_name()) - seg1.write("one fish two fish ") - seg2.write("red fish blue fish") - - manifest = self.env.container.file("manifest" + Utils.create_name()) - manifest.write( - '', - hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" % - (container2.name,)}) - - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(manifest.path), - self.env.tempurl_key) - parms = {'temp_url_sig': sig, - 'temp_url_expires': str(expires)} - - # cross container tempurl does not work for container tempurl key - try: - manifest.read(parms=parms, cfg={'no_auth_token': True}) - except ResponseError as e: - self.assertEqual(e.status, 401) - else: - self.fail('request did not error') - try: - manifest.info(parms=parms, cfg={'no_auth_token': True}) - except ResponseError as e: - self.assertEqual(e.status, 401) - else: - self.fail('request did not error') - - -class TestContainerTempurlUTF8(Base2, TestContainerTempurl): - set_up = False - - -class TestSloTempurlEnv(object): - enabled = None # tri-state: None initially, then True/False - - @classmethod - def setUp(cls): - cls.conn = Connection(tf.config) - cls.conn.authenticate() - - if cls.enabled is None: - cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info - - cls.tempurl_key = Utils.create_name() - - cls.account = Account( - cls.conn, tf.config.get('account', tf.config['username'])) - cls.account.delete_containers() - cls.account.update_metadata({'temp-url-key': cls.tempurl_key}) - - cls.manifest_container = cls.account.container(Utils.create_name()) - cls.segments_container = cls.account.container(Utils.create_name()) - if not cls.manifest_container.create(): - raise ResponseError(cls.conn.response) - if not cls.segments_container.create(): - raise ResponseError(cls.conn.response) - - seg1 = cls.segments_container.file(Utils.create_name()) - seg1.write('1' * 1024 * 1024) - - seg2 = cls.segments_container.file(Utils.create_name()) - seg2.write('2' * 1024 * 1024) - - cls.manifest_data = [{'size_bytes': 1024 * 1024, - 'etag': seg1.md5, - 'path': '/%s/%s' % (cls.segments_container.name, - seg1.name)}, - {'size_bytes': 1024 * 1024, - 'etag': seg2.md5, - 'path': '/%s/%s' % (cls.segments_container.name, - seg2.name)}] - - cls.manifest = cls.manifest_container.file(Utils.create_name()) - cls.manifest.write( - json.dumps(cls.manifest_data), - parms={'multipart-manifest': 'put'}) - - -class TestSloTempurl(Base): - env = TestSloTempurlEnv - set_up = False - - def setUp(self): - super(TestSloTempurl, self).setUp() - if self.env.enabled is False: - raise SkipTest("TempURL and SLO not both enabled") - elif self.env.enabled is not True: - # just some sanity checking - raise Exception( - "Expected enabled to be True/False, got %r" % - (self.env.enabled,)) - - def tempurl_sig(self, method, expires, path, key): - return hmac.new( - key, - '%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)), - hashlib.sha1).hexdigest() - - def test_GET(self): - expires = int(time.time()) + 86400 - sig = self.tempurl_sig( - 'GET', expires, self.env.conn.make_path(self.env.manifest.path), - self.env.tempurl_key) - parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)} - - contents = self.env.manifest.read( - parms=parms, - cfg={'no_auth_token': True}) - self.assertEqual(len(contents), 2 * 1024 * 1024) - - # GET tempurls also allow HEAD requests - self.assertTrue(self.env.manifest.info( - parms=parms, cfg={'no_auth_token': True})) - - -class TestSloTempurlUTF8(Base2, TestSloTempurl): - set_up = False - - class TestServiceToken(unittest2.TestCase): def setUp(self):