Merge "breaking up functional/tests.py a bit further"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
18b392b602
396
test/functional/test_dlo.py
Normal file
396
test/functional/test_dlo.py
Normal file
@ -0,0 +1,396 @@
|
||||
#!/usr/bin/python -u
|
||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import test.functional as tf
|
||||
from test.functional.tests import Utils, Base, Base2
|
||||
from test.functional.swift_test_client import Account, Connection, \
|
||||
ResponseError
|
||||
|
||||
|
||||
def setUpModule():
|
||||
tf.setup_package()
|
||||
|
||||
|
||||
def tearDownModule():
|
||||
tf.teardown_package()
|
||||
|
||||
|
||||
class TestDloEnv(object):
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.conn = Connection(tf.config)
|
||||
cls.conn.authenticate()
|
||||
|
||||
config2 = tf.config.copy()
|
||||
config2['username'] = tf.config['username3']
|
||||
config2['password'] = tf.config['password3']
|
||||
cls.conn2 = Connection(config2)
|
||||
cls.conn2.authenticate()
|
||||
|
||||
cls.account = Account(cls.conn, tf.config.get('account',
|
||||
tf.config['username']))
|
||||
cls.account.delete_containers()
|
||||
|
||||
cls.container = cls.account.container(Utils.create_name())
|
||||
cls.container2 = cls.account.container(Utils.create_name())
|
||||
|
||||
for cont in (cls.container, cls.container2):
|
||||
if not cont.create():
|
||||
raise ResponseError(cls.conn.response)
|
||||
|
||||
# avoid getting a prefix that stops halfway through an encoded
|
||||
# character
|
||||
prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
|
||||
cls.segment_prefix = prefix
|
||||
|
||||
for letter in ('a', 'b', 'c', 'd', 'e'):
|
||||
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
|
||||
file_item.write(letter * 10)
|
||||
|
||||
file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
|
||||
file_item.write(letter.upper() * 10)
|
||||
|
||||
for letter in ('f', 'g', 'h', 'i', 'j'):
|
||||
file_item = cls.container2.file("%s/seg_lower%s" %
|
||||
(prefix, letter))
|
||||
file_item.write(letter * 10)
|
||||
|
||||
man1 = cls.container.file("man1")
|
||||
man1.write('man1-contents',
|
||||
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
|
||||
(cls.container.name, prefix)})
|
||||
|
||||
man2 = cls.container.file("man2")
|
||||
man2.write('man2-contents',
|
||||
hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
|
||||
(cls.container.name, prefix)})
|
||||
|
||||
manall = cls.container.file("manall")
|
||||
manall.write('manall-contents',
|
||||
hdrs={"X-Object-Manifest": "%s/%s/seg" %
|
||||
(cls.container.name, prefix)})
|
||||
|
||||
mancont2 = cls.container.file("mancont2")
|
||||
mancont2.write(
|
||||
'mancont2-contents',
|
||||
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
|
||||
(cls.container2.name, prefix)})
|
||||
|
||||
|
||||
class TestDlo(Base):
|
||||
env = TestDloEnv
|
||||
set_up = False
|
||||
|
||||
def test_get_manifest(self):
|
||||
file_item = self.env.container.file('man1')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(
|
||||
file_contents,
|
||||
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
|
||||
|
||||
file_item = self.env.container.file('man2')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(
|
||||
file_contents,
|
||||
"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")
|
||||
|
||||
file_item = self.env.container.file('manall')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(
|
||||
file_contents,
|
||||
("aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
|
||||
"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))
|
||||
|
||||
def test_get_manifest_document_itself(self):
|
||||
file_item = self.env.container.file('man1')
|
||||
file_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(file_contents, "man1-contents")
|
||||
self.assertEqual(file_item.info()['x_object_manifest'],
|
||||
"%s/%s/seg_lower" %
|
||||
(self.env.container.name, self.env.segment_prefix))
|
||||
|
||||
def test_get_range(self):
|
||||
file_item = self.env.container.file('man1')
|
||||
file_contents = file_item.read(size=25, offset=8)
|
||||
self.assertEqual(file_contents, "aabbbbbbbbbbccccccccccddd")
|
||||
|
||||
file_contents = file_item.read(size=1, offset=47)
|
||||
self.assertEqual(file_contents, "e")
|
||||
|
||||
def test_get_range_out_of_range(self):
|
||||
file_item = self.env.container.file('man1')
|
||||
|
||||
self.assertRaises(ResponseError, file_item.read, size=7, offset=50)
|
||||
self.assert_status(416)
|
||||
|
||||
def test_copy(self):
|
||||
# Adding a new segment, copying the manifest, and then deleting the
|
||||
# segment proves that the new object is really the concatenated
|
||||
# segments and not just a manifest.
|
||||
f_segment = self.env.container.file("%s/seg_lowerf" %
|
||||
(self.env.segment_prefix))
|
||||
f_segment.write('ffffffffff')
|
||||
try:
|
||||
man1_item = self.env.container.file('man1')
|
||||
man1_item.copy(self.env.container.name, "copied-man1")
|
||||
finally:
|
||||
# try not to leave this around for other tests to stumble over
|
||||
f_segment.delete()
|
||||
|
||||
file_item = self.env.container.file('copied-man1')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(
|
||||
file_contents,
|
||||
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
|
||||
# The copied object must not have X-Object-Manifest
|
||||
self.assertNotIn("x_object_manifest", file_item.info())
|
||||
|
||||
def test_copy_account(self):
|
||||
# dlo use same account and same container only
|
||||
acct = self.env.conn.account_name
|
||||
# Adding a new segment, copying the manifest, and then deleting the
|
||||
# segment proves that the new object is really the concatenated
|
||||
# segments and not just a manifest.
|
||||
f_segment = self.env.container.file("%s/seg_lowerf" %
|
||||
(self.env.segment_prefix))
|
||||
f_segment.write('ffffffffff')
|
||||
try:
|
||||
man1_item = self.env.container.file('man1')
|
||||
man1_item.copy_account(acct,
|
||||
self.env.container.name,
|
||||
"copied-man1")
|
||||
finally:
|
||||
# try not to leave this around for other tests to stumble over
|
||||
f_segment.delete()
|
||||
|
||||
file_item = self.env.container.file('copied-man1')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(
|
||||
file_contents,
|
||||
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
|
||||
# The copied object must not have X-Object-Manifest
|
||||
self.assertNotIn("x_object_manifest", file_item.info())
|
||||
|
||||
def test_copy_manifest(self):
|
||||
# Copying the manifest with multipart-manifest=get query string
|
||||
# should result in another manifest
|
||||
try:
|
||||
man1_item = self.env.container.file('man1')
|
||||
man1_item.copy(self.env.container.name, "copied-man1",
|
||||
parms={'multipart-manifest': 'get'})
|
||||
|
||||
copied = self.env.container.file("copied-man1")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(copied_contents, "man1-contents")
|
||||
|
||||
copied_contents = copied.read()
|
||||
self.assertEqual(
|
||||
copied_contents,
|
||||
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
|
||||
self.assertEqual(man1_item.info()['x_object_manifest'],
|
||||
copied.info()['x_object_manifest'])
|
||||
finally:
|
||||
# try not to leave this around for other tests to stumble over
|
||||
self.env.container.file("copied-man1").delete()
|
||||
|
||||
def test_dlo_if_match_get(self):
|
||||
manifest = self.env.container.file("man1")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.read,
|
||||
hdrs={'If-Match': 'not-%s' % etag})
|
||||
self.assert_status(412)
|
||||
|
||||
manifest.read(hdrs={'If-Match': etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_dlo_if_none_match_get(self):
|
||||
manifest = self.env.container.file("man1")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.read,
|
||||
hdrs={'If-None-Match': etag})
|
||||
self.assert_status(304)
|
||||
|
||||
manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_dlo_if_match_head(self):
|
||||
manifest = self.env.container.file("man1")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.info,
|
||||
hdrs={'If-Match': 'not-%s' % etag})
|
||||
self.assert_status(412)
|
||||
|
||||
manifest.info(hdrs={'If-Match': etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_dlo_if_none_match_head(self):
|
||||
manifest = self.env.container.file("man1")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.info,
|
||||
hdrs={'If-None-Match': etag})
|
||||
self.assert_status(304)
|
||||
|
||||
manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_dlo_referer_on_segment_container(self):
|
||||
# First the account2 (test3) should fail
|
||||
headers = {'X-Auth-Token': self.env.conn2.storage_token,
|
||||
'Referer': 'http://blah.example.com'}
|
||||
dlo_file = self.env.container.file("mancont2")
|
||||
self.assertRaises(ResponseError, dlo_file.read,
|
||||
hdrs=headers)
|
||||
self.assert_status(403)
|
||||
|
||||
# Now set the referer on the dlo container only
|
||||
referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
|
||||
self.env.container.update_metadata(referer_metadata)
|
||||
|
||||
self.assertRaises(ResponseError, dlo_file.read,
|
||||
hdrs=headers)
|
||||
self.assert_status(403)
|
||||
|
||||
# Finally set the referer on the segment container
|
||||
self.env.container2.update_metadata(referer_metadata)
|
||||
|
||||
contents = dlo_file.read(hdrs=headers)
|
||||
self.assertEqual(
|
||||
contents,
|
||||
"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
|
||||
|
||||
def test_dlo_post_with_manifest_header(self):
|
||||
# verify that performing a POST to a DLO manifest
|
||||
# preserves the fact that it is a manifest file.
|
||||
# verify that the x-object-manifest header may be updated.
|
||||
|
||||
# create a new manifest for this test to avoid test coupling.
|
||||
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
||||
file_item = self.env.container.file(Utils.create_name())
|
||||
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
|
||||
|
||||
# sanity checks
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('manifest-contents', manifest_contents)
|
||||
expected_contents = ''.join([(c * 10) for c in 'abcde'])
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual(expected_contents, contents)
|
||||
|
||||
# POST a modified x-object-manifest value
|
||||
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
|
||||
file_item.post({'x-object-meta-foo': 'bar',
|
||||
'x-object-manifest': new_x_o_m})
|
||||
|
||||
# verify that x-object-manifest was updated
|
||||
file_item.info()
|
||||
resp_headers = file_item.conn.response.getheaders()
|
||||
self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
|
||||
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
|
||||
|
||||
# verify that manifest content was not changed
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('manifest-contents', manifest_contents)
|
||||
|
||||
# verify that updated manifest points to new content
|
||||
expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual(expected_contents, contents)
|
||||
|
||||
# Now revert the manifest to point to original segments, including a
|
||||
# multipart-manifest=get param just to check that has no effect
|
||||
file_item.post({'x-object-manifest': x_o_m},
|
||||
parms={'multipart-manifest': 'get'})
|
||||
|
||||
# verify that x-object-manifest was reverted
|
||||
info = file_item.info()
|
||||
self.assertIn('x_object_manifest', info)
|
||||
self.assertEqual(x_o_m, info['x_object_manifest'])
|
||||
|
||||
# verify that manifest content was not changed
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('manifest-contents', manifest_contents)
|
||||
|
||||
# verify that updated manifest points new content
|
||||
expected_contents = ''.join([(c * 10) for c in 'abcde'])
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual(expected_contents, contents)
|
||||
|
||||
def test_dlo_post_without_manifest_header(self):
|
||||
# verify that a POST to a DLO manifest object with no
|
||||
# x-object-manifest header will cause the existing x-object-manifest
|
||||
# header to be lost
|
||||
|
||||
# create a new manifest for this test to avoid test coupling.
|
||||
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
||||
file_item = self.env.container.file(Utils.create_name())
|
||||
file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})
|
||||
|
||||
# sanity checks
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('manifest-contents', manifest_contents)
|
||||
expected_contents = ''.join([(c * 10) for c in 'abcde'])
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual(expected_contents, contents)
|
||||
|
||||
# POST with no x-object-manifest header
|
||||
file_item.post({})
|
||||
|
||||
# verify that existing x-object-manifest was removed
|
||||
info = file_item.info()
|
||||
self.assertNotIn('x_object_manifest', info)
|
||||
|
||||
# verify that object content was not changed
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('manifest-contents', manifest_contents)
|
||||
|
||||
# verify that object is no longer a manifest
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual('manifest-contents', contents)
|
||||
|
||||
def test_dlo_post_with_manifest_regular_object(self):
|
||||
# verify that performing a POST to a regular object
|
||||
# with a manifest header will create a DLO.
|
||||
|
||||
# Put a regular object
|
||||
file_item = self.env.container.file(Utils.create_name())
|
||||
file_item.write('file contents', hdrs={})
|
||||
|
||||
# sanity checks
|
||||
file_contents = file_item.read(parms={})
|
||||
self.assertEqual('file contents', file_contents)
|
||||
|
||||
# get the path associated with man1
|
||||
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
||||
|
||||
# POST a x-object-manifest value to the regular object
|
||||
file_item.post({'x-object-manifest': x_o_m})
|
||||
|
||||
# verify that the file is now a manifest
|
||||
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual('file contents', manifest_contents)
|
||||
expected_contents = ''.join([(c * 10) for c in 'abcde'])
|
||||
contents = file_item.read(parms={})
|
||||
self.assertEqual(expected_contents, contents)
|
||||
file_item.info()
|
||||
resp_headers = file_item.conn.response.getheaders()
|
||||
self.assertIn(('x-object-manifest', x_o_m), resp_headers)
|
||||
|
||||
|
||||
class TestDloUTF8(Base2, TestDlo):
|
||||
set_up = False
|
||||
938
test/functional/test_slo.py
Normal file
938
test/functional/test_slo.py
Normal file
@ -0,0 +1,938 @@
|
||||
#!/usr/bin/python -u
|
||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import email.parser
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from unittest2 import SkipTest
|
||||
|
||||
import test.functional as tf
|
||||
from test.functional import cluster_info
|
||||
from test.functional.tests import Utils, Base, Base2
|
||||
from test.functional.swift_test_client import Account, Connection, \
|
||||
ResponseError
|
||||
|
||||
|
||||
def setUpModule():
|
||||
tf.setup_package()
|
||||
|
||||
|
||||
def tearDownModule():
|
||||
tf.teardown_package()
|
||||
|
||||
|
||||
class TestSloEnv(object):
|
||||
slo_enabled = None # tri-state: None initially, then True/False
|
||||
|
||||
@classmethod
|
||||
def create_segments(cls, container):
|
||||
seg_info = {}
|
||||
for letter, size in (('a', 1024 * 1024),
|
||||
('b', 1024 * 1024),
|
||||
('c', 1024 * 1024),
|
||||
('d', 1024 * 1024),
|
||||
('e', 1)):
|
||||
seg_name = "seg_%s" % letter
|
||||
file_item = container.file(seg_name)
|
||||
file_item.write(letter * size)
|
||||
seg_info[seg_name] = {
|
||||
'size_bytes': size,
|
||||
'etag': file_item.md5,
|
||||
'path': '/%s/%s' % (container.name, seg_name)}
|
||||
return seg_info
|
||||
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.conn = Connection(tf.config)
|
||||
cls.conn.authenticate()
|
||||
config2 = deepcopy(tf.config)
|
||||
config2['account'] = tf.config['account2']
|
||||
config2['username'] = tf.config['username2']
|
||||
config2['password'] = tf.config['password2']
|
||||
cls.conn2 = Connection(config2)
|
||||
cls.conn2.authenticate()
|
||||
cls.account2 = cls.conn2.get_account()
|
||||
cls.account2.delete_containers()
|
||||
config3 = tf.config.copy()
|
||||
config3['username'] = tf.config['username3']
|
||||
config3['password'] = tf.config['password3']
|
||||
cls.conn3 = Connection(config3)
|
||||
cls.conn3.authenticate()
|
||||
|
||||
if cls.slo_enabled is None:
|
||||
cls.slo_enabled = 'slo' in cluster_info
|
||||
if not cls.slo_enabled:
|
||||
return
|
||||
|
||||
cls.account = Account(cls.conn, tf.config.get('account',
|
||||
tf.config['username']))
|
||||
cls.account.delete_containers()
|
||||
|
||||
cls.container = cls.account.container(Utils.create_name())
|
||||
cls.container2 = cls.account.container(Utils.create_name())
|
||||
|
||||
for cont in (cls.container, cls.container2):
|
||||
if not cont.create():
|
||||
raise ResponseError(cls.conn.response)
|
||||
|
||||
cls.seg_info = seg_info = cls.create_segments(cls.container)
|
||||
|
||||
file_item = cls.container.file("manifest-abcde")
|
||||
file_item.write(
|
||||
json.dumps([seg_info['seg_a'], seg_info['seg_b'],
|
||||
seg_info['seg_c'], seg_info['seg_d'],
|
||||
seg_info['seg_e']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
|
||||
# Put the same manifest in the container2
|
||||
file_item = cls.container2.file("manifest-abcde")
|
||||
file_item.write(
|
||||
json.dumps([seg_info['seg_a'], seg_info['seg_b'],
|
||||
seg_info['seg_c'], seg_info['seg_d'],
|
||||
seg_info['seg_e']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
|
||||
file_item = cls.container.file('manifest-cd')
|
||||
cd_json = json.dumps([seg_info['seg_c'], seg_info['seg_d']])
|
||||
file_item.write(cd_json, parms={'multipart-manifest': 'put'})
|
||||
cd_etag = hashlib.md5(seg_info['seg_c']['etag'] +
|
||||
seg_info['seg_d']['etag']).hexdigest()
|
||||
|
||||
file_item = cls.container.file("manifest-bcd-submanifest")
|
||||
file_item.write(
|
||||
json.dumps([seg_info['seg_b'],
|
||||
{'etag': cd_etag,
|
||||
'size_bytes': (seg_info['seg_c']['size_bytes'] +
|
||||
seg_info['seg_d']['size_bytes']),
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'manifest-cd')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
bcd_submanifest_etag = hashlib.md5(
|
||||
seg_info['seg_b']['etag'] + cd_etag).hexdigest()
|
||||
|
||||
file_item = cls.container.file("manifest-abcde-submanifest")
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
seg_info['seg_a'],
|
||||
{'etag': bcd_submanifest_etag,
|
||||
'size_bytes': (seg_info['seg_b']['size_bytes'] +
|
||||
seg_info['seg_c']['size_bytes'] +
|
||||
seg_info['seg_d']['size_bytes']),
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'manifest-bcd-submanifest')},
|
||||
seg_info['seg_e']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
abcde_submanifest_etag = hashlib.md5(
|
||||
seg_info['seg_a']['etag'] + bcd_submanifest_etag +
|
||||
seg_info['seg_e']['etag']).hexdigest()
|
||||
abcde_submanifest_size = (seg_info['seg_a']['size_bytes'] +
|
||||
seg_info['seg_b']['size_bytes'] +
|
||||
seg_info['seg_c']['size_bytes'] +
|
||||
seg_info['seg_d']['size_bytes'] +
|
||||
seg_info['seg_e']['size_bytes'])
|
||||
|
||||
file_item = cls.container.file("ranged-manifest")
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
{'etag': abcde_submanifest_etag,
|
||||
'size_bytes': abcde_submanifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'manifest-abcde-submanifest'),
|
||||
'range': '-1048578'}, # 'c' + ('d' * 2**20) + 'e'
|
||||
{'etag': abcde_submanifest_etag,
|
||||
'size_bytes': abcde_submanifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'manifest-abcde-submanifest'),
|
||||
'range': '524288-1572863'}, # 'a' * 2**19 + 'b' * 2**19
|
||||
{'etag': abcde_submanifest_etag,
|
||||
'size_bytes': abcde_submanifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'manifest-abcde-submanifest'),
|
||||
'range': '3145727-3145728'}]), # 'cd'
|
||||
parms={'multipart-manifest': 'put'})
|
||||
ranged_manifest_etag = hashlib.md5(
|
||||
abcde_submanifest_etag + ':3145727-4194304;' +
|
||||
abcde_submanifest_etag + ':524288-1572863;' +
|
||||
abcde_submanifest_etag + ':3145727-3145728;').hexdigest()
|
||||
ranged_manifest_size = 2 * 1024 * 1024 + 4
|
||||
|
||||
file_item = cls.container.file("ranged-submanifest")
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
seg_info['seg_c'],
|
||||
{'etag': ranged_manifest_etag,
|
||||
'size_bytes': ranged_manifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'ranged-manifest')},
|
||||
{'etag': ranged_manifest_etag,
|
||||
'size_bytes': ranged_manifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'ranged-manifest'),
|
||||
'range': '524289-1572865'},
|
||||
{'etag': ranged_manifest_etag,
|
||||
'size_bytes': ranged_manifest_size,
|
||||
'path': '/%s/%s' % (cls.container.name,
|
||||
'ranged-manifest'),
|
||||
'range': '-3'}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
|
||||
file_item = cls.container.file("manifest-db")
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
{'path': seg_info['seg_d']['path'], 'etag': None,
|
||||
'size_bytes': None},
|
||||
{'path': seg_info['seg_b']['path'], 'etag': None,
|
||||
'size_bytes': None},
|
||||
]), parms={'multipart-manifest': 'put'})
|
||||
|
||||
file_item = cls.container.file("ranged-manifest-repeated-segment")
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
{'path': seg_info['seg_a']['path'], 'etag': None,
|
||||
'size_bytes': None, 'range': '-1048578'},
|
||||
{'path': seg_info['seg_a']['path'], 'etag': None,
|
||||
'size_bytes': None},
|
||||
{'path': seg_info['seg_b']['path'], 'etag': None,
|
||||
'size_bytes': None, 'range': '-1048578'},
|
||||
]), parms={'multipart-manifest': 'put'})
|
||||
|
||||
|
||||
class TestSlo(Base):
|
||||
env = TestSloEnv
|
||||
set_up = False
|
||||
|
||||
def setUp(self):
|
||||
super(TestSlo, self).setUp()
|
||||
if self.env.slo_enabled is False:
|
||||
raise SkipTest("SLO not enabled")
|
||||
elif self.env.slo_enabled is not True:
|
||||
# just some sanity checking
|
||||
raise Exception(
|
||||
"Expected slo_enabled to be True/False, got %r" %
|
||||
(self.env.slo_enabled,))
|
||||
|
||||
def test_slo_get_simple_manifest(self):
|
||||
file_item = self.env.container.file('manifest-abcde')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
|
||||
self.assertEqual('a', file_contents[0])
|
||||
self.assertEqual('a', file_contents[1024 * 1024 - 1])
|
||||
self.assertEqual('b', file_contents[1024 * 1024])
|
||||
self.assertEqual('d', file_contents[-2])
|
||||
self.assertEqual('e', file_contents[-1])
|
||||
|
||||
def test_slo_container_listing(self):
|
||||
# the listing object size should equal the sum of the size of the
|
||||
# segments, not the size of the manifest body
|
||||
file_item = self.env.container.file(Utils.create_name())
|
||||
file_item.write(
|
||||
json.dumps([self.env.seg_info['seg_a']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
# The container listing has the etag of the actual manifest object
|
||||
# contents which we get using multipart-manifest=get. Arguably this
|
||||
# should be the etag that we get when NOT using multipart-manifest=get,
|
||||
# to be consistent with size and content-type. But here we at least
|
||||
# verify that it remains consistent when the object is updated with a
|
||||
# POST.
|
||||
file_item.initialize(parms={'multipart-manifest': 'get'})
|
||||
expected_etag = file_item.etag
|
||||
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual('application/octet-stream',
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
|
||||
# now POST updated content-type file
|
||||
file_item.content_type = 'image/jpeg'
|
||||
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'})
|
||||
file_item.initialize()
|
||||
self.assertEqual('image/jpeg', file_item.content_type) # sanity
|
||||
|
||||
# verify that the container listing is consistent with the file
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual(file_item.content_type,
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
|
||||
# now POST with no change to content-type
|
||||
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'},
|
||||
cfg={'no_content_type': True})
|
||||
file_item.initialize()
|
||||
self.assertEqual('image/jpeg', file_item.content_type) # sanity
|
||||
|
||||
# verify that the container listing is consistent with the file
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual(file_item.content_type,
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
|
||||
def test_slo_get_nested_manifest(self):
|
||||
file_item = self.env.container.file('manifest-abcde-submanifest')
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(file_contents))
|
||||
self.assertEqual('a', file_contents[0])
|
||||
self.assertEqual('a', file_contents[1024 * 1024 - 1])
|
||||
self.assertEqual('b', file_contents[1024 * 1024])
|
||||
self.assertEqual('d', file_contents[-2])
|
||||
self.assertEqual('e', file_contents[-1])
|
||||
|
||||
def test_slo_get_ranged_manifest(self):
|
||||
file_item = self.env.container.file('ranged-manifest')
|
||||
grouped_file_contents = [
|
||||
(char, sum(1 for _char in grp))
|
||||
for char, grp in itertools.groupby(file_item.read())]
|
||||
self.assertEqual([
|
||||
('c', 1),
|
||||
('d', 1024 * 1024),
|
||||
('e', 1),
|
||||
('a', 512 * 1024),
|
||||
('b', 512 * 1024),
|
||||
('c', 1),
|
||||
('d', 1)], grouped_file_contents)
|
||||
|
||||
def test_slo_get_ranged_manifest_repeated_segment(self):
|
||||
file_item = self.env.container.file('ranged-manifest-repeated-segment')
|
||||
grouped_file_contents = [
|
||||
(char, sum(1 for _char in grp))
|
||||
for char, grp in itertools.groupby(file_item.read())]
|
||||
self.assertEqual(
|
||||
[('a', 2097152), ('b', 1048576)],
|
||||
grouped_file_contents)
|
||||
|
||||
def test_slo_get_ranged_submanifest(self):
|
||||
file_item = self.env.container.file('ranged-submanifest')
|
||||
grouped_file_contents = [
|
||||
(char, sum(1 for _char in grp))
|
||||
for char, grp in itertools.groupby(file_item.read())]
|
||||
self.assertEqual([
|
||||
('c', 1024 * 1024 + 1),
|
||||
('d', 1024 * 1024),
|
||||
('e', 1),
|
||||
('a', 512 * 1024),
|
||||
('b', 512 * 1024),
|
||||
('c', 1),
|
||||
('d', 512 * 1024 + 1),
|
||||
('e', 1),
|
||||
('a', 512 * 1024),
|
||||
('b', 1),
|
||||
('c', 1),
|
||||
('d', 1)], grouped_file_contents)
|
||||
|
||||
def test_slo_ranged_get(self):
|
||||
file_item = self.env.container.file('manifest-abcde')
|
||||
file_contents = file_item.read(size=1024 * 1024 + 2,
|
||||
offset=1024 * 1024 - 1)
|
||||
self.assertEqual('a', file_contents[0])
|
||||
self.assertEqual('b', file_contents[1])
|
||||
self.assertEqual('b', file_contents[-2])
|
||||
self.assertEqual('c', file_contents[-1])
|
||||
|
||||
def test_slo_multi_ranged_get(self):
|
||||
file_item = self.env.container.file('manifest-abcde')
|
||||
file_contents = file_item.read(
|
||||
hdrs={"Range": "bytes=1048571-1048580,2097147-2097156"})
|
||||
|
||||
# See testMultiRangeGets for explanation
|
||||
parser = email.parser.FeedParser()
|
||||
parser.feed("Content-Type: %s\r\n\r\n" % file_item.content_type)
|
||||
parser.feed(file_contents)
|
||||
|
||||
root_message = parser.close()
|
||||
self.assertTrue(root_message.is_multipart()) # sanity check
|
||||
|
||||
byteranges = root_message.get_payload()
|
||||
self.assertEqual(len(byteranges), 2)
|
||||
|
||||
self.assertEqual(byteranges[0]['Content-Type'],
|
||||
"application/octet-stream")
|
||||
self.assertEqual(
|
||||
byteranges[0]['Content-Range'], "bytes 1048571-1048580/4194305")
|
||||
self.assertEqual(byteranges[0].get_payload(), "aaaaabbbbb")
|
||||
|
||||
self.assertEqual(byteranges[1]['Content-Type'],
|
||||
"application/octet-stream")
|
||||
self.assertEqual(
|
||||
byteranges[1]['Content-Range'], "bytes 2097147-2097156/4194305")
|
||||
self.assertEqual(byteranges[1].get_payload(), "bbbbbccccc")
|
||||
|
||||
def test_slo_ranged_submanifest(self):
|
||||
file_item = self.env.container.file('manifest-abcde-submanifest')
|
||||
file_contents = file_item.read(size=1024 * 1024 + 2,
|
||||
offset=1024 * 1024 * 2 - 1)
|
||||
self.assertEqual('b', file_contents[0])
|
||||
self.assertEqual('c', file_contents[1])
|
||||
self.assertEqual('c', file_contents[-2])
|
||||
self.assertEqual('d', file_contents[-1])
|
||||
|
||||
def test_slo_etag_is_hash_of_etags(self):
|
||||
expected_hash = hashlib.md5()
|
||||
expected_hash.update(hashlib.md5('a' * 1024 * 1024).hexdigest())
|
||||
expected_hash.update(hashlib.md5('b' * 1024 * 1024).hexdigest())
|
||||
expected_hash.update(hashlib.md5('c' * 1024 * 1024).hexdigest())
|
||||
expected_hash.update(hashlib.md5('d' * 1024 * 1024).hexdigest())
|
||||
expected_hash.update(hashlib.md5('e').hexdigest())
|
||||
expected_etag = expected_hash.hexdigest()
|
||||
|
||||
file_item = self.env.container.file('manifest-abcde')
|
||||
self.assertEqual(expected_etag, file_item.info()['etag'])
|
||||
|
||||
def test_slo_etag_is_hash_of_etags_submanifests(self):
|
||||
|
||||
def hd(x):
|
||||
return hashlib.md5(x).hexdigest()
|
||||
|
||||
expected_etag = hd(hd('a' * 1024 * 1024) +
|
||||
hd(hd('b' * 1024 * 1024) +
|
||||
hd(hd('c' * 1024 * 1024) +
|
||||
hd('d' * 1024 * 1024))) +
|
||||
hd('e'))
|
||||
|
||||
file_item = self.env.container.file('manifest-abcde-submanifest')
|
||||
self.assertEqual(expected_etag, file_item.info()['etag'])
|
||||
|
||||
def test_slo_etag_mismatch(self):
|
||||
file_item = self.env.container.file("manifest-a-bad-etag")
|
||||
try:
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'size_bytes': 1024 * 1024,
|
||||
'etag': 'not it',
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
except ResponseError as err:
|
||||
self.assertEqual(400, err.status)
|
||||
else:
|
||||
self.fail("Expected ResponseError but didn't get it")
|
||||
|
||||
def test_slo_size_mismatch(self):
|
||||
file_item = self.env.container.file("manifest-a-bad-size")
|
||||
try:
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'size_bytes': 1024 * 1024 - 1,
|
||||
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
except ResponseError as err:
|
||||
self.assertEqual(400, err.status)
|
||||
else:
|
||||
self.fail("Expected ResponseError but didn't get it")
|
||||
|
||||
def test_slo_unspecified_etag(self):
|
||||
file_item = self.env.container.file("manifest-a-unspecified-etag")
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'size_bytes': 1024 * 1024,
|
||||
'etag': None,
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
self.assert_status(201)
|
||||
|
||||
def test_slo_unspecified_size(self):
|
||||
file_item = self.env.container.file("manifest-a-unspecified-size")
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'size_bytes': None,
|
||||
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
self.assert_status(201)
|
||||
|
||||
def test_slo_missing_etag(self):
|
||||
file_item = self.env.container.file("manifest-a-missing-etag")
|
||||
try:
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'size_bytes': 1024 * 1024,
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
except ResponseError as err:
|
||||
self.assertEqual(400, err.status)
|
||||
else:
|
||||
self.fail("Expected ResponseError but didn't get it")
|
||||
|
||||
def test_slo_missing_size(self):
|
||||
file_item = self.env.container.file("manifest-a-missing-size")
|
||||
try:
|
||||
file_item.write(
|
||||
json.dumps([{
|
||||
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
except ResponseError as err:
|
||||
self.assertEqual(400, err.status)
|
||||
else:
|
||||
self.fail("Expected ResponseError but didn't get it")
|
||||
|
||||
def test_slo_overwrite_segment_with_manifest(self):
|
||||
file_item = self.env.container.file("seg_b")
|
||||
with self.assertRaises(ResponseError) as catcher:
|
||||
file_item.write(
|
||||
json.dumps([
|
||||
{'size_bytes': 1024 * 1024,
|
||||
'etag': hashlib.md5('a' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')},
|
||||
{'size_bytes': 1024 * 1024,
|
||||
'etag': hashlib.md5('b' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_b')},
|
||||
{'size_bytes': 1024 * 1024,
|
||||
'etag': hashlib.md5('c' * 1024 * 1024).hexdigest(),
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_c')}]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
self.assertEqual(400, catcher.exception.status)
|
||||
|
||||
def test_slo_copy(self):
|
||||
file_item = self.env.container.file("manifest-abcde")
|
||||
file_item.copy(self.env.container.name, "copied-abcde")
|
||||
|
||||
copied = self.env.container.file("copied-abcde")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
|
||||
|
||||
def test_slo_copy_account(self):
|
||||
acct = self.env.conn.account_name
|
||||
# same account copy
|
||||
file_item = self.env.container.file("manifest-abcde")
|
||||
file_item.copy_account(acct, self.env.container.name, "copied-abcde")
|
||||
|
||||
copied = self.env.container.file("copied-abcde")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
|
||||
|
||||
# copy to different account
|
||||
acct = self.env.conn2.account_name
|
||||
dest_cont = self.env.account2.container(Utils.create_name())
|
||||
self.assertTrue(dest_cont.create(hdrs={
|
||||
'X-Container-Write': self.env.conn.user_acl
|
||||
}))
|
||||
file_item = self.env.container.file("manifest-abcde")
|
||||
file_item.copy_account(acct, dest_cont, "copied-abcde")
|
||||
|
||||
copied = dest_cont.file("copied-abcde")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
|
||||
|
||||
def test_slo_copy_the_manifest(self):
|
||||
source = self.env.container.file("manifest-abcde")
|
||||
source_contents = source.read(parms={'multipart-manifest': 'get'})
|
||||
source_json = json.loads(source_contents)
|
||||
source.initialize()
|
||||
self.assertEqual('application/octet-stream', source.content_type)
|
||||
source.initialize(parms={'multipart-manifest': 'get'})
|
||||
source_hash = hashlib.md5()
|
||||
source_hash.update(source_contents)
|
||||
self.assertEqual(source_hash.hexdigest(), source.etag)
|
||||
|
||||
self.assertTrue(source.copy(self.env.container.name,
|
||||
"copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'}))
|
||||
|
||||
copied = self.env.container.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
copied_json = json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
self.assertEqual(source_json, copied_json)
|
||||
copied.initialize()
|
||||
self.assertEqual('application/octet-stream', copied.content_type)
|
||||
copied.initialize(parms={'multipart-manifest': 'get'})
|
||||
copied_hash = hashlib.md5()
|
||||
copied_hash.update(copied_contents)
|
||||
self.assertEqual(copied_hash.hexdigest(), copied.etag)
|
||||
|
||||
# verify the listing metadata
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
names = {}
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] in ('manifest-abcde',
|
||||
'copied-abcde-manifest-only'):
|
||||
names[f_dict['name']] = f_dict
|
||||
|
||||
self.assertIn('manifest-abcde', names)
|
||||
actual = names['manifest-abcde']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
self.assertEqual(source.etag, actual['hash'])
|
||||
|
||||
self.assertIn('copied-abcde-manifest-only', names)
|
||||
actual = names['copied-abcde-manifest-only']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
self.assertEqual(copied.etag, actual['hash'])
|
||||
|
||||
def test_slo_copy_the_manifest_updating_metadata(self):
|
||||
source = self.env.container.file("manifest-abcde")
|
||||
source.content_type = 'application/octet-stream'
|
||||
source.sync_metadata({'test': 'original'})
|
||||
source_contents = source.read(parms={'multipart-manifest': 'get'})
|
||||
source_json = json.loads(source_contents)
|
||||
source.initialize()
|
||||
self.assertEqual('application/octet-stream', source.content_type)
|
||||
source.initialize(parms={'multipart-manifest': 'get'})
|
||||
source_hash = hashlib.md5()
|
||||
source_hash.update(source_contents)
|
||||
self.assertEqual(source_hash.hexdigest(), source.etag)
|
||||
self.assertEqual(source.metadata['test'], 'original')
|
||||
|
||||
self.assertTrue(
|
||||
source.copy(self.env.container.name, "copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'},
|
||||
hdrs={'Content-Type': 'image/jpeg',
|
||||
'X-Object-Meta-Test': 'updated'}))
|
||||
|
||||
copied = self.env.container.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
copied_json = json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
self.assertEqual(source_json, copied_json)
|
||||
copied.initialize()
|
||||
self.assertEqual('image/jpeg', copied.content_type)
|
||||
copied.initialize(parms={'multipart-manifest': 'get'})
|
||||
copied_hash = hashlib.md5()
|
||||
copied_hash.update(copied_contents)
|
||||
self.assertEqual(copied_hash.hexdigest(), copied.etag)
|
||||
self.assertEqual(copied.metadata['test'], 'updated')
|
||||
|
||||
# verify the listing metadata
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
names = {}
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] in ('manifest-abcde',
|
||||
'copied-abcde-manifest-only'):
|
||||
names[f_dict['name']] = f_dict
|
||||
|
||||
self.assertIn('manifest-abcde', names)
|
||||
actual = names['manifest-abcde']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
# the container listing should have the etag of the manifest contents
|
||||
self.assertEqual(source.etag, actual['hash'])
|
||||
|
||||
self.assertIn('copied-abcde-manifest-only', names)
|
||||
actual = names['copied-abcde-manifest-only']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('image/jpeg', actual['content_type'])
|
||||
self.assertEqual(copied.etag, actual['hash'])
|
||||
|
||||
def test_slo_copy_the_manifest_account(self):
|
||||
acct = self.env.conn.account_name
|
||||
# same account
|
||||
file_item = self.env.container.file("manifest-abcde")
|
||||
file_item.copy_account(acct,
|
||||
self.env.container.name,
|
||||
"copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'})
|
||||
|
||||
copied = self.env.container.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
|
||||
# different account
|
||||
acct = self.env.conn2.account_name
|
||||
dest_cont = self.env.account2.container(Utils.create_name())
|
||||
self.assertTrue(dest_cont.create(hdrs={
|
||||
'X-Container-Write': self.env.conn.user_acl
|
||||
}))
|
||||
|
||||
# manifest copy will fail because there is no read access to segments
|
||||
# in destination account
|
||||
file_item.copy_account(
|
||||
acct, dest_cont, "copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(400, file_item.conn.response.status)
|
||||
resp_body = file_item.conn.response.read()
|
||||
self.assertEqual(5, resp_body.count('403 Forbidden'),
|
||||
'Unexpected response body %r' % resp_body)
|
||||
|
||||
# create segments container in account2 with read access for account1
|
||||
segs_container = self.env.account2.container(self.env.container.name)
|
||||
self.assertTrue(segs_container.create(hdrs={
|
||||
'X-Container-Read': self.env.conn.user_acl
|
||||
}))
|
||||
|
||||
# manifest copy will still fail because there are no segments in
|
||||
# destination account
|
||||
file_item.copy_account(
|
||||
acct, dest_cont, "copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'})
|
||||
self.assertEqual(400, file_item.conn.response.status)
|
||||
resp_body = file_item.conn.response.read()
|
||||
self.assertEqual(5, resp_body.count('404 Not Found'),
|
||||
'Unexpected response body %r' % resp_body)
|
||||
|
||||
# create segments in account2 container with same name as in account1,
|
||||
# manifest copy now succeeds
|
||||
self.env.create_segments(segs_container)
|
||||
|
||||
self.assertTrue(file_item.copy_account(
|
||||
acct, dest_cont, "copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'}))
|
||||
|
||||
copied = dest_cont.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
|
||||
def _make_manifest(self):
|
||||
file_item = self.env.container.file("manifest-post")
|
||||
seg_info = self.env.seg_info
|
||||
file_item.write(
|
||||
json.dumps([seg_info['seg_a'], seg_info['seg_b'],
|
||||
seg_info['seg_c'], seg_info['seg_d'],
|
||||
seg_info['seg_e']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
return file_item
|
||||
|
||||
def test_slo_post_the_manifest_metadata_update(self):
|
||||
file_item = self._make_manifest()
|
||||
# sanity check, check the object is an SLO manifest
|
||||
file_item.info()
|
||||
file_item.header_fields([('slo', 'x-static-large-object')])
|
||||
|
||||
# POST a user metadata (i.e. x-object-meta-post)
|
||||
file_item.sync_metadata({'post': 'update'})
|
||||
|
||||
updated = self.env.container.file("manifest-post")
|
||||
updated.info()
|
||||
updated.header_fields([('user-meta', 'x-object-meta-post')]) # sanity
|
||||
updated.header_fields([('slo', 'x-static-large-object')])
|
||||
updated_contents = updated.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
json.loads(updated_contents)
|
||||
except ValueError:
|
||||
self.fail("Unexpected content on GET, expected a json body")
|
||||
|
||||
def test_slo_post_the_manifest_metadata_update_with_qs(self):
|
||||
# multipart-manifest query should be ignored on post
|
||||
for verb in ('put', 'get', 'delete'):
|
||||
file_item = self._make_manifest()
|
||||
# sanity check, check the object is an SLO manifest
|
||||
file_item.info()
|
||||
file_item.header_fields([('slo', 'x-static-large-object')])
|
||||
# POST a user metadata (i.e. x-object-meta-post)
|
||||
file_item.sync_metadata(metadata={'post': 'update'},
|
||||
parms={'multipart-manifest': verb})
|
||||
updated = self.env.container.file("manifest-post")
|
||||
updated.info()
|
||||
updated.header_fields(
|
||||
[('user-meta', 'x-object-meta-post')]) # sanity
|
||||
updated.header_fields([('slo', 'x-static-large-object')])
|
||||
updated_contents = updated.read(
|
||||
parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
json.loads(updated_contents)
|
||||
except ValueError:
|
||||
self.fail(
|
||||
"Unexpected content on GET, expected a json body")
|
||||
|
||||
def test_slo_get_the_manifest(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
got_body = manifest.read(parms={'multipart-manifest': 'get'})
|
||||
|
||||
self.assertEqual('application/json; charset=utf-8',
|
||||
manifest.content_type)
|
||||
try:
|
||||
json.loads(got_body)
|
||||
except ValueError:
|
||||
self.fail("GET with multipart-manifest=get got invalid json")
|
||||
|
||||
def test_slo_get_the_manifest_with_details_from_server(self):
|
||||
manifest = self.env.container.file("manifest-db")
|
||||
got_body = manifest.read(parms={'multipart-manifest': 'get'})
|
||||
|
||||
self.assertEqual('application/json; charset=utf-8',
|
||||
manifest.content_type)
|
||||
try:
|
||||
value = json.loads(got_body)
|
||||
except ValueError:
|
||||
self.fail("GET with multipart-manifest=get got invalid json")
|
||||
|
||||
self.assertEqual(len(value), 2)
|
||||
self.assertEqual(value[0]['bytes'], 1024 * 1024)
|
||||
self.assertEqual(value[0]['hash'],
|
||||
hashlib.md5('d' * 1024 * 1024).hexdigest())
|
||||
self.assertEqual(value[0]['name'],
|
||||
'/%s/seg_d' % self.env.container.name.decode("utf-8"))
|
||||
|
||||
self.assertEqual(value[1]['bytes'], 1024 * 1024)
|
||||
self.assertEqual(value[1]['hash'],
|
||||
hashlib.md5('b' * 1024 * 1024).hexdigest())
|
||||
self.assertEqual(value[1]['name'],
|
||||
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
|
||||
|
||||
def test_slo_get_raw_the_manifest_with_details_from_server(self):
|
||||
manifest = self.env.container.file("manifest-db")
|
||||
got_body = manifest.read(parms={'multipart-manifest': 'get',
|
||||
'format': 'raw'})
|
||||
|
||||
# raw format should have the actual manifest object content-type
|
||||
self.assertEqual('application/octet-stream', manifest.content_type)
|
||||
try:
|
||||
value = json.loads(got_body)
|
||||
except ValueError:
|
||||
msg = "GET with multipart-manifest=get&format=raw got invalid json"
|
||||
self.fail(msg)
|
||||
|
||||
self.assertEqual(
|
||||
set(value[0].keys()), set(('size_bytes', 'etag', 'path')))
|
||||
self.assertEqual(len(value), 2)
|
||||
self.assertEqual(value[0]['size_bytes'], 1024 * 1024)
|
||||
self.assertEqual(value[0]['etag'],
|
||||
hashlib.md5('d' * 1024 * 1024).hexdigest())
|
||||
self.assertEqual(value[0]['path'],
|
||||
'/%s/seg_d' % self.env.container.name.decode("utf-8"))
|
||||
self.assertEqual(value[1]['size_bytes'], 1024 * 1024)
|
||||
self.assertEqual(value[1]['etag'],
|
||||
hashlib.md5('b' * 1024 * 1024).hexdigest())
|
||||
self.assertEqual(value[1]['path'],
|
||||
'/%s/seg_b' % self.env.container.name.decode("utf-8"))
|
||||
|
||||
file_item = self.env.container.file("manifest-from-get-raw")
|
||||
file_item.write(got_body, parms={'multipart-manifest': 'put'})
|
||||
|
||||
file_contents = file_item.read()
|
||||
self.assertEqual(2 * 1024 * 1024, len(file_contents))
|
||||
|
||||
def test_slo_head_the_manifest(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
got_info = manifest.info(parms={'multipart-manifest': 'get'})
|
||||
|
||||
self.assertEqual('application/json; charset=utf-8',
|
||||
got_info['content_type'])
|
||||
|
||||
def test_slo_if_match_get(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.read,
|
||||
hdrs={'If-Match': 'not-%s' % etag})
|
||||
self.assert_status(412)
|
||||
|
||||
manifest.read(hdrs={'If-Match': etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_slo_if_none_match_put(self):
|
||||
file_item = self.env.container.file("manifest-if-none-match")
|
||||
manifest = json.dumps([{
|
||||
'size_bytes': 1024 * 1024,
|
||||
'etag': None,
|
||||
'path': '/%s/%s' % (self.env.container.name, 'seg_a')}])
|
||||
|
||||
self.assertRaises(ResponseError, file_item.write, manifest,
|
||||
parms={'multipart-manifest': 'put'},
|
||||
hdrs={'If-None-Match': '"not-star"'})
|
||||
self.assert_status(400)
|
||||
|
||||
file_item.write(manifest, parms={'multipart-manifest': 'put'},
|
||||
hdrs={'If-None-Match': '*'})
|
||||
self.assert_status(201)
|
||||
|
||||
self.assertRaises(ResponseError, file_item.write, manifest,
|
||||
parms={'multipart-manifest': 'put'},
|
||||
hdrs={'If-None-Match': '*'})
|
||||
self.assert_status(412)
|
||||
|
||||
def test_slo_if_none_match_get(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.read,
|
||||
hdrs={'If-None-Match': etag})
|
||||
self.assert_status(304)
|
||||
|
||||
manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_slo_if_match_head(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.info,
|
||||
hdrs={'If-Match': 'not-%s' % etag})
|
||||
self.assert_status(412)
|
||||
|
||||
manifest.info(hdrs={'If-Match': etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_slo_if_none_match_head(self):
|
||||
manifest = self.env.container.file("manifest-abcde")
|
||||
etag = manifest.info()['etag']
|
||||
|
||||
self.assertRaises(ResponseError, manifest.info,
|
||||
hdrs={'If-None-Match': etag})
|
||||
self.assert_status(304)
|
||||
|
||||
manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
|
||||
self.assert_status(200)
|
||||
|
||||
def test_slo_referer_on_segment_container(self):
|
||||
# First the account2 (test3) should fail
|
||||
headers = {'X-Auth-Token': self.env.conn3.storage_token,
|
||||
'Referer': 'http://blah.example.com'}
|
||||
slo_file = self.env.container2.file('manifest-abcde')
|
||||
self.assertRaises(ResponseError, slo_file.read,
|
||||
hdrs=headers)
|
||||
self.assert_status(403)
|
||||
|
||||
# Now set the referer on the slo container only
|
||||
referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
|
||||
self.env.container2.update_metadata(referer_metadata)
|
||||
|
||||
self.assertRaises(ResponseError, slo_file.read,
|
||||
hdrs=headers)
|
||||
self.assert_status(409)
|
||||
|
||||
# Finally set the referer on the segment container
|
||||
self.env.container.update_metadata(referer_metadata)
|
||||
contents = slo_file.read(hdrs=headers)
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(contents))
|
||||
self.assertEqual('a', contents[0])
|
||||
self.assertEqual('a', contents[1024 * 1024 - 1])
|
||||
self.assertEqual('b', contents[1024 * 1024])
|
||||
self.assertEqual('d', contents[-2])
|
||||
self.assertEqual('e', contents[-1])
|
||||
|
||||
|
||||
class TestSloUTF8(Base2, TestSlo):
|
||||
set_up = False
|
||||
674
test/functional/test_tempurl.py
Normal file
674
test/functional/test_tempurl.py
Normal file
@ -0,0 +1,674 @@
|
||||
#!/usr/bin/python -u
|
||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import hmac
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from copy import deepcopy
|
||||
from six.moves import urllib
|
||||
from unittest2 import SkipTest
|
||||
|
||||
import test.functional as tf
|
||||
from test.functional import cluster_info
|
||||
from test.functional.tests import Utils, Base, Base2
|
||||
from test.functional import requires_acls
|
||||
from test.functional.swift_test_client import Account, Connection, \
|
||||
ResponseError
|
||||
|
||||
|
||||
def setUpModule():
|
||||
tf.setup_package()
|
||||
|
||||
|
||||
def tearDownModule():
|
||||
tf.teardown_package()
|
||||
|
||||
|
||||
class TestTempurlEnv(object):
|
||||
tempurl_enabled = None # tri-state: None initially, then True/False
|
||||
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.conn = Connection(tf.config)
|
||||
cls.conn.authenticate()
|
||||
|
||||
if cls.tempurl_enabled is None:
|
||||
cls.tempurl_enabled = 'tempurl' in cluster_info
|
||||
if not cls.tempurl_enabled:
|
||||
return
|
||||
|
||||
cls.tempurl_key = Utils.create_name()
|
||||
cls.tempurl_key2 = Utils.create_name()
|
||||
|
||||
cls.account = Account(
|
||||
cls.conn, tf.config.get('account', tf.config['username']))
|
||||
cls.account.delete_containers()
|
||||
cls.account.update_metadata({
|
||||
'temp-url-key': cls.tempurl_key,
|
||||
'temp-url-key-2': cls.tempurl_key2
|
||||
})
|
||||
|
||||
cls.container = cls.account.container(Utils.create_name())
|
||||
if not cls.container.create():
|
||||
raise ResponseError(cls.conn.response)
|
||||
|
||||
cls.obj = cls.container.file(Utils.create_name())
|
||||
cls.obj.write("obj contents")
|
||||
cls.other_obj = cls.container.file(Utils.create_name())
|
||||
cls.other_obj.write("other obj contents")
|
||||
|
||||
|
||||
class TestTempurl(Base):
|
||||
env = TestTempurlEnv
|
||||
set_up = False
|
||||
|
||||
def setUp(self):
|
||||
super(TestTempurl, self).setUp()
|
||||
if self.env.tempurl_enabled is False:
|
||||
raise SkipTest("TempURL not enabled")
|
||||
elif self.env.tempurl_enabled is not True:
|
||||
# just some sanity checking
|
||||
raise Exception(
|
||||
"Expected tempurl_enabled to be True/False, got %r" %
|
||||
(self.env.tempurl_enabled,))
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key)
|
||||
self.obj_tempurl_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
def tempurl_sig(self, method, expires, path, key):
|
||||
return hmac.new(
|
||||
key,
|
||||
'%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)),
|
||||
hashlib.sha1).hexdigest()
|
||||
|
||||
def test_GET(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
# GET tempurls also allow HEAD requests
|
||||
self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
|
||||
def test_GET_with_key_2(self):
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key2)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
def test_GET_DLO_inside_container(self):
|
||||
seg1 = self.env.container.file(
|
||||
"get-dlo-inside-seg1" + Utils.create_name())
|
||||
seg2 = self.env.container.file(
|
||||
"get-dlo-inside-seg2" + Utils.create_name())
|
||||
seg1.write("one fish two fish ")
|
||||
seg2.write("red fish blue fish")
|
||||
|
||||
manifest = self.env.container.file("manifest" + Utils.create_name())
|
||||
manifest.write(
|
||||
'',
|
||||
hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
|
||||
(self.env.container.name,)})
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(manifest.path),
|
||||
self.env.tempurl_key)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "one fish two fish red fish blue fish")
|
||||
|
||||
def test_GET_DLO_outside_container(self):
|
||||
seg1 = self.env.container.file(
|
||||
"get-dlo-outside-seg1" + Utils.create_name())
|
||||
seg2 = self.env.container.file(
|
||||
"get-dlo-outside-seg2" + Utils.create_name())
|
||||
seg1.write("one fish two fish ")
|
||||
seg2.write("red fish blue fish")
|
||||
|
||||
container2 = self.env.account.container(Utils.create_name())
|
||||
container2.create()
|
||||
|
||||
manifest = container2.file("manifest" + Utils.create_name())
|
||||
manifest.write(
|
||||
'',
|
||||
hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
|
||||
(self.env.container.name,)})
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(manifest.path),
|
||||
self.env.tempurl_key)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
# cross container tempurl works fine for account tempurl key
|
||||
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "one fish two fish red fish blue fish")
|
||||
self.assert_status([200])
|
||||
|
||||
def test_PUT(self):
|
||||
new_obj = self.env.container.file(Utils.create_name())
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'PUT', expires, self.env.conn.make_path(new_obj.path),
|
||||
self.env.tempurl_key)
|
||||
put_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
new_obj.write('new obj contents',
|
||||
parms=put_parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(new_obj.read(), "new obj contents")
|
||||
|
||||
# PUT tempurls also allow HEAD requests
|
||||
self.assertTrue(new_obj.info(parms=put_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
|
||||
def test_PUT_manifest_access(self):
|
||||
new_obj = self.env.container.file(Utils.create_name())
|
||||
|
||||
# give out a signature which allows a PUT to new_obj
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'PUT', expires, self.env.conn.make_path(new_obj.path),
|
||||
self.env.tempurl_key)
|
||||
put_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
# try to create manifest pointing to some random container
|
||||
try:
|
||||
new_obj.write('', {
|
||||
'x-object-manifest': '%s/foo' % 'some_random_container'
|
||||
}, parms=put_parms, cfg={'no_auth_token': True})
|
||||
except ResponseError as e:
|
||||
self.assertEqual(e.status, 400)
|
||||
else:
|
||||
self.fail('request did not error')
|
||||
|
||||
# create some other container
|
||||
other_container = self.env.account.container(Utils.create_name())
|
||||
if not other_container.create():
|
||||
raise ResponseError(self.conn.response)
|
||||
|
||||
# try to create manifest pointing to new container
|
||||
try:
|
||||
new_obj.write('', {
|
||||
'x-object-manifest': '%s/foo' % other_container
|
||||
}, parms=put_parms, cfg={'no_auth_token': True})
|
||||
except ResponseError as e:
|
||||
self.assertEqual(e.status, 400)
|
||||
else:
|
||||
self.fail('request did not error')
|
||||
|
||||
# try again using a tempurl POST to an already created object
|
||||
new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True})
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'POST', expires, self.env.conn.make_path(new_obj.path),
|
||||
self.env.tempurl_key)
|
||||
post_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
try:
|
||||
new_obj.post({'x-object-manifest': '%s/foo' % other_container},
|
||||
parms=post_parms, cfg={'no_auth_token': True})
|
||||
except ResponseError as e:
|
||||
self.assertEqual(e.status, 400)
|
||||
else:
|
||||
self.fail('request did not error')
|
||||
|
||||
def test_HEAD(self):
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'HEAD', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key)
|
||||
head_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
self.assertTrue(self.env.obj.info(parms=head_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
# HEAD tempurls don't allow PUT or GET requests, despite the fact that
|
||||
# PUT and GET tempurls both allow HEAD requests
|
||||
self.assertRaises(ResponseError, self.env.other_obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
self.assertRaises(ResponseError, self.env.other_obj.write,
|
||||
'new contents',
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_different_object(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
self.assertRaises(ResponseError, self.env.other_obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_changing_sig(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
parms = self.obj_tempurl_parms.copy()
|
||||
if parms['temp_url_sig'][0] == 'a':
|
||||
parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:]
|
||||
else:
|
||||
parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:]
|
||||
|
||||
self.assertRaises(ResponseError, self.env.obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_changing_expires(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
parms = self.obj_tempurl_parms.copy()
|
||||
if parms['temp_url_expires'][-1] == '0':
|
||||
parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1'
|
||||
else:
|
||||
parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0'
|
||||
|
||||
self.assertRaises(ResponseError, self.env.obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=parms)
|
||||
self.assert_status([401])
|
||||
|
||||
|
||||
class TestTempurlUTF8(Base2, TestTempurl):
|
||||
set_up = False
|
||||
|
||||
|
||||
class TestContainerTempurlEnv(object):
|
||||
tempurl_enabled = None # tri-state: None initially, then True/False
|
||||
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.conn = Connection(tf.config)
|
||||
cls.conn.authenticate()
|
||||
|
||||
if cls.tempurl_enabled is None:
|
||||
cls.tempurl_enabled = 'tempurl' in cluster_info
|
||||
if not cls.tempurl_enabled:
|
||||
return
|
||||
|
||||
cls.tempurl_key = Utils.create_name()
|
||||
cls.tempurl_key2 = Utils.create_name()
|
||||
|
||||
cls.account = Account(
|
||||
cls.conn, tf.config.get('account', tf.config['username']))
|
||||
cls.account.delete_containers()
|
||||
|
||||
# creating another account and connection
|
||||
# for ACL tests
|
||||
config2 = deepcopy(tf.config)
|
||||
config2['account'] = tf.config['account2']
|
||||
config2['username'] = tf.config['username2']
|
||||
config2['password'] = tf.config['password2']
|
||||
cls.conn2 = Connection(config2)
|
||||
cls.conn2.authenticate()
|
||||
cls.account2 = Account(
|
||||
cls.conn2, config2.get('account', config2['username']))
|
||||
cls.account2 = cls.conn2.get_account()
|
||||
|
||||
cls.container = cls.account.container(Utils.create_name())
|
||||
if not cls.container.create({
|
||||
'x-container-meta-temp-url-key': cls.tempurl_key,
|
||||
'x-container-meta-temp-url-key-2': cls.tempurl_key2,
|
||||
'x-container-read': cls.account2.name}):
|
||||
raise ResponseError(cls.conn.response)
|
||||
|
||||
cls.obj = cls.container.file(Utils.create_name())
|
||||
cls.obj.write("obj contents")
|
||||
cls.other_obj = cls.container.file(Utils.create_name())
|
||||
cls.other_obj.write("other obj contents")
|
||||
|
||||
|
||||
class TestContainerTempurl(Base):
|
||||
env = TestContainerTempurlEnv
|
||||
set_up = False
|
||||
|
||||
def setUp(self):
|
||||
super(TestContainerTempurl, self).setUp()
|
||||
if self.env.tempurl_enabled is False:
|
||||
raise SkipTest("TempURL not enabled")
|
||||
elif self.env.tempurl_enabled is not True:
|
||||
# just some sanity checking
|
||||
raise Exception(
|
||||
"Expected tempurl_enabled to be True/False, got %r" %
|
||||
(self.env.tempurl_enabled,))
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key)
|
||||
self.obj_tempurl_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
def tempurl_sig(self, method, expires, path, key):
|
||||
return hmac.new(
|
||||
key,
|
||||
'%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)),
|
||||
hashlib.sha1).hexdigest()
|
||||
|
||||
def test_GET(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
# GET tempurls also allow HEAD requests
|
||||
self.assertTrue(self.env.obj.info(parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
|
||||
def test_GET_with_key_2(self):
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key2)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
contents = self.env.obj.read(parms=parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
def test_PUT(self):
|
||||
new_obj = self.env.container.file(Utils.create_name())
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'PUT', expires, self.env.conn.make_path(new_obj.path),
|
||||
self.env.tempurl_key)
|
||||
put_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
new_obj.write('new obj contents',
|
||||
parms=put_parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(new_obj.read(), "new obj contents")
|
||||
|
||||
# PUT tempurls also allow HEAD requests
|
||||
self.assertTrue(new_obj.info(parms=put_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
|
||||
def test_HEAD(self):
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'HEAD', expires, self.env.conn.make_path(self.env.obj.path),
|
||||
self.env.tempurl_key)
|
||||
head_parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
self.assertTrue(self.env.obj.info(parms=head_parms,
|
||||
cfg={'no_auth_token': True}))
|
||||
# HEAD tempurls don't allow PUT or GET requests, despite the fact that
|
||||
# PUT and GET tempurls both allow HEAD requests
|
||||
self.assertRaises(ResponseError, self.env.other_obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
self.assertRaises(ResponseError, self.env.other_obj.write,
|
||||
'new contents',
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_different_object(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
self.assertRaises(ResponseError, self.env.other_obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=self.obj_tempurl_parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_changing_sig(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
parms = self.obj_tempurl_parms.copy()
|
||||
if parms['temp_url_sig'][0] == 'a':
|
||||
parms['temp_url_sig'] = 'b' + parms['temp_url_sig'][1:]
|
||||
else:
|
||||
parms['temp_url_sig'] = 'a' + parms['temp_url_sig'][1:]
|
||||
|
||||
self.assertRaises(ResponseError, self.env.obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=parms)
|
||||
self.assert_status([401])
|
||||
|
||||
def test_changing_expires(self):
|
||||
contents = self.env.obj.read(
|
||||
parms=self.obj_tempurl_parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "obj contents")
|
||||
|
||||
parms = self.obj_tempurl_parms.copy()
|
||||
if parms['temp_url_expires'][-1] == '0':
|
||||
parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '1'
|
||||
else:
|
||||
parms['temp_url_expires'] = parms['temp_url_expires'][:-1] + '0'
|
||||
|
||||
self.assertRaises(ResponseError, self.env.obj.read,
|
||||
cfg={'no_auth_token': True},
|
||||
parms=parms)
|
||||
self.assert_status([401])
|
||||
|
||||
@requires_acls
|
||||
def test_tempurl_keys_visible_to_account_owner(self):
|
||||
if not tf.cluster_info.get('tempauth'):
|
||||
raise SkipTest('TEMP AUTH SPECIFIC TEST')
|
||||
metadata = self.env.container.info()
|
||||
self.assertEqual(metadata.get('tempurl_key'), self.env.tempurl_key)
|
||||
self.assertEqual(metadata.get('tempurl_key2'), self.env.tempurl_key2)
|
||||
|
||||
@requires_acls
|
||||
def test_tempurl_keys_hidden_from_acl_readonly(self):
|
||||
if not tf.cluster_info.get('tempauth'):
|
||||
raise SkipTest('TEMP AUTH SPECIFIC TEST')
|
||||
original_token = self.env.container.conn.storage_token
|
||||
self.env.container.conn.storage_token = self.env.conn2.storage_token
|
||||
metadata = self.env.container.info()
|
||||
self.env.container.conn.storage_token = original_token
|
||||
|
||||
self.assertNotIn(
|
||||
'tempurl_key', metadata,
|
||||
'Container TempURL key found, should not be visible '
|
||||
'to readonly ACLs')
|
||||
self.assertNotIn(
|
||||
'tempurl_key2', metadata,
|
||||
'Container TempURL key-2 found, should not be visible '
|
||||
'to readonly ACLs')
|
||||
|
||||
def test_GET_DLO_inside_container(self):
|
||||
seg1 = self.env.container.file(
|
||||
"get-dlo-inside-seg1" + Utils.create_name())
|
||||
seg2 = self.env.container.file(
|
||||
"get-dlo-inside-seg2" + Utils.create_name())
|
||||
seg1.write("one fish two fish ")
|
||||
seg2.write("red fish blue fish")
|
||||
|
||||
manifest = self.env.container.file("manifest" + Utils.create_name())
|
||||
manifest.write(
|
||||
'',
|
||||
hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
|
||||
(self.env.container.name,)})
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(manifest.path),
|
||||
self.env.tempurl_key)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
|
||||
self.assertEqual(contents, "one fish two fish red fish blue fish")
|
||||
|
||||
def test_GET_DLO_outside_container(self):
|
||||
container2 = self.env.account.container(Utils.create_name())
|
||||
container2.create()
|
||||
seg1 = container2.file(
|
||||
"get-dlo-outside-seg1" + Utils.create_name())
|
||||
seg2 = container2.file(
|
||||
"get-dlo-outside-seg2" + Utils.create_name())
|
||||
seg1.write("one fish two fish ")
|
||||
seg2.write("red fish blue fish")
|
||||
|
||||
manifest = self.env.container.file("manifest" + Utils.create_name())
|
||||
manifest.write(
|
||||
'',
|
||||
hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
|
||||
(container2.name,)})
|
||||
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(manifest.path),
|
||||
self.env.tempurl_key)
|
||||
parms = {'temp_url_sig': sig,
|
||||
'temp_url_expires': str(expires)}
|
||||
|
||||
# cross container tempurl does not work for container tempurl key
|
||||
try:
|
||||
manifest.read(parms=parms, cfg={'no_auth_token': True})
|
||||
except ResponseError as e:
|
||||
self.assertEqual(e.status, 401)
|
||||
else:
|
||||
self.fail('request did not error')
|
||||
try:
|
||||
manifest.info(parms=parms, cfg={'no_auth_token': True})
|
||||
except ResponseError as e:
|
||||
self.assertEqual(e.status, 401)
|
||||
else:
|
||||
self.fail('request did not error')
|
||||
|
||||
|
||||
class TestContainerTempurlUTF8(Base2, TestContainerTempurl):
|
||||
set_up = False
|
||||
|
||||
|
||||
class TestSloTempurlEnv(object):
|
||||
enabled = None # tri-state: None initially, then True/False
|
||||
|
||||
@classmethod
|
||||
def setUp(cls):
|
||||
cls.conn = Connection(tf.config)
|
||||
cls.conn.authenticate()
|
||||
|
||||
if cls.enabled is None:
|
||||
cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info
|
||||
|
||||
cls.tempurl_key = Utils.create_name()
|
||||
|
||||
cls.account = Account(
|
||||
cls.conn, tf.config.get('account', tf.config['username']))
|
||||
cls.account.delete_containers()
|
||||
cls.account.update_metadata({'temp-url-key': cls.tempurl_key})
|
||||
|
||||
cls.manifest_container = cls.account.container(Utils.create_name())
|
||||
cls.segments_container = cls.account.container(Utils.create_name())
|
||||
if not cls.manifest_container.create():
|
||||
raise ResponseError(cls.conn.response)
|
||||
if not cls.segments_container.create():
|
||||
raise ResponseError(cls.conn.response)
|
||||
|
||||
seg1 = cls.segments_container.file(Utils.create_name())
|
||||
seg1.write('1' * 1024 * 1024)
|
||||
|
||||
seg2 = cls.segments_container.file(Utils.create_name())
|
||||
seg2.write('2' * 1024 * 1024)
|
||||
|
||||
cls.manifest_data = [{'size_bytes': 1024 * 1024,
|
||||
'etag': seg1.md5,
|
||||
'path': '/%s/%s' % (cls.segments_container.name,
|
||||
seg1.name)},
|
||||
{'size_bytes': 1024 * 1024,
|
||||
'etag': seg2.md5,
|
||||
'path': '/%s/%s' % (cls.segments_container.name,
|
||||
seg2.name)}]
|
||||
|
||||
cls.manifest = cls.manifest_container.file(Utils.create_name())
|
||||
cls.manifest.write(
|
||||
json.dumps(cls.manifest_data),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
|
||||
|
||||
class TestSloTempurl(Base):
|
||||
env = TestSloTempurlEnv
|
||||
set_up = False
|
||||
|
||||
def setUp(self):
|
||||
super(TestSloTempurl, self).setUp()
|
||||
if self.env.enabled is False:
|
||||
raise SkipTest("TempURL and SLO not both enabled")
|
||||
elif self.env.enabled is not True:
|
||||
# just some sanity checking
|
||||
raise Exception(
|
||||
"Expected enabled to be True/False, got %r" %
|
||||
(self.env.enabled,))
|
||||
|
||||
def tempurl_sig(self, method, expires, path, key):
|
||||
return hmac.new(
|
||||
key,
|
||||
'%s\n%s\n%s' % (method, expires, urllib.parse.unquote(path)),
|
||||
hashlib.sha1).hexdigest()
|
||||
|
||||
def test_GET(self):
|
||||
expires = int(time.time()) + 86400
|
||||
sig = self.tempurl_sig(
|
||||
'GET', expires, self.env.conn.make_path(self.env.manifest.path),
|
||||
self.env.tempurl_key)
|
||||
parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)}
|
||||
|
||||
contents = self.env.manifest.read(
|
||||
parms=parms,
|
||||
cfg={'no_auth_token': True})
|
||||
self.assertEqual(len(contents), 2 * 1024 * 1024)
|
||||
|
||||
# GET tempurls also allow HEAD requests
|
||||
self.assertTrue(self.env.manifest.info(
|
||||
parms=parms, cfg={'no_auth_token': True}))
|
||||
|
||||
|
||||
class TestSloTempurlUTF8(Base2, TestSloTempurl):
|
||||
set_up = False
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user