407 lines
16 KiB
Python
407 lines
16 KiB
Python
#!/usr/bin/python -u
|
|
# Copyright (c) 2010-2016 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from swift.common.swob import str_to_wsgi
|
|
import test.functional as tf
|
|
from test.functional.tests import Utils, Base, Base2, BaseEnv
|
|
from test.functional.swift_test_client import Connection, ResponseError
|
|
|
|
|
|
def setUpModule():
|
|
tf.setup_package()
|
|
|
|
|
|
def tearDownModule():
|
|
tf.teardown_package()
|
|
|
|
|
|
class TestDloEnv(BaseEnv):
|
|
@classmethod
|
|
def setUp(cls):
|
|
super(TestDloEnv, cls).setUp()
|
|
|
|
cls.container = cls.account.container(Utils.create_name())
|
|
cls.container2 = cls.account.container(Utils.create_name())
|
|
|
|
for cont in (cls.container, cls.container2):
|
|
if not cont.create():
|
|
raise ResponseError(cls.conn.response)
|
|
|
|
prefix = Utils.create_name(10)
|
|
cls.segment_prefix = prefix
|
|
|
|
for letter in ('a', 'b', 'c', 'd', 'e'):
|
|
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
|
|
file_item.write(letter.encode('ascii') * 10)
|
|
|
|
file_item = cls.container.file(
|
|
"%s/seg_upper_%%ff%s" % (prefix, letter))
|
|
file_item.write(letter.upper().encode('ascii') * 10)
|
|
|
|
for letter in ('f', 'g', 'h', 'i', 'j'):
|
|
file_item = cls.container2.file("%s/seg_lower%s" %
|
|
(prefix, letter))
|
|
file_item.write(letter.encode('ascii') * 10)
|
|
|
|
man1 = cls.container.file("man1")
|
|
man1.write(b'man1-contents',
|
|
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
|
|
(cls.container.name, prefix)})
|
|
|
|
man2 = cls.container.file("man2")
|
|
man2.write(b'man2-contents',
|
|
hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
|
|
(cls.container.name, prefix)})
|
|
|
|
manall = cls.container.file("manall")
|
|
manall.write(b'manall-contents',
|
|
hdrs={"X-Object-Manifest": "%s/%s/seg" %
|
|
(cls.container.name, prefix)})
|
|
|
|
mancont2 = cls.container.file("mancont2")
|
|
mancont2.write(
|
|
b'mancont2-contents',
|
|
hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
|
|
(cls.container2.name, prefix)})
|
|
|
|
|
|
class TestDlo(Base):
|
|
env = TestDloEnv
|
|
|
|
def test_get_manifest(self):
|
|
file_item = self.env.container.file('man1')
|
|
file_contents = file_item.read()
|
|
self.assertEqual(
|
|
file_contents,
|
|
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
|
|
|
|
file_item = self.env.container.file('man2')
|
|
file_contents = file_item.read()
|
|
self.assertEqual(
|
|
file_contents,
|
|
b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE")
|
|
|
|
file_item = self.env.container.file('manall')
|
|
file_contents = file_item.read()
|
|
self.assertEqual(
|
|
file_contents,
|
|
(b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee" +
|
|
b"AAAAAAAAAABBBBBBBBBBCCCCCCCCCCDDDDDDDDDDEEEEEEEEEE"))
|
|
|
|
def test_get_manifest_document_itself(self):
|
|
file_item = self.env.container.file('man1')
|
|
file_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(file_contents, b"man1-contents")
|
|
self.assertEqual(file_item.info()['x_object_manifest'],
|
|
"%s/%s/seg_lower" %
|
|
(self.env.container.name, self.env.segment_prefix))
|
|
|
|
def test_get_range(self):
|
|
file_item = self.env.container.file('man1')
|
|
file_contents = file_item.read(size=25, offset=8)
|
|
self.assertEqual(file_contents, b"aabbbbbbbbbbccccccccccddd")
|
|
|
|
file_contents = file_item.read(size=1, offset=47)
|
|
self.assertEqual(file_contents, b"e")
|
|
|
|
def test_get_multiple_ranges(self):
|
|
file_item = self.env.container.file('man1')
|
|
file_contents = file_item.read(
|
|
hdrs={'Range': 'bytes=0-4,10-14'})
|
|
self.assert_status(200) # *not* 206
|
|
self.assertEqual(
|
|
file_contents,
|
|
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
|
|
|
|
def test_get_range_out_of_range(self):
|
|
file_item = self.env.container.file('man1')
|
|
|
|
self.assertRaises(ResponseError, file_item.read, size=7, offset=50)
|
|
self.assert_status(416)
|
|
self.assert_header('content-range', 'bytes */50')
|
|
|
|
def test_copy(self):
|
|
# Adding a new segment, copying the manifest, and then deleting the
|
|
# segment proves that the new object is really the concatenated
|
|
# segments and not just a manifest.
|
|
f_segment = self.env.container.file("%s/seg_lowerf" %
|
|
(self.env.segment_prefix))
|
|
f_segment.write(b'ffffffffff')
|
|
try:
|
|
man1_item = self.env.container.file('man1')
|
|
man1_item.copy(self.env.container.name, "copied-man1")
|
|
finally:
|
|
# try not to leave this around for other tests to stumble over
|
|
f_segment.delete(tolerate_missing=True)
|
|
|
|
file_item = self.env.container.file('copied-man1')
|
|
file_contents = file_item.read()
|
|
self.assertEqual(
|
|
file_contents,
|
|
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
|
|
# The copied object must not have X-Object-Manifest
|
|
self.assertNotIn("x_object_manifest", file_item.info())
|
|
|
|
def test_copy_account(self):
|
|
# dlo use same account and same container only
|
|
acct = self.env.conn.account_name
|
|
# Adding a new segment, copying the manifest, and then deleting the
|
|
# segment proves that the new object is really the concatenated
|
|
# segments and not just a manifest.
|
|
f_segment = self.env.container.file("%s/seg_lowerf" %
|
|
(self.env.segment_prefix))
|
|
f_segment.write(b'ffffffffff')
|
|
try:
|
|
man1_item = self.env.container.file('man1')
|
|
man1_item.copy_account(acct,
|
|
self.env.container.name,
|
|
"copied-man1")
|
|
finally:
|
|
# try not to leave this around for other tests to stumble over
|
|
f_segment.delete(tolerate_missing=True)
|
|
|
|
file_item = self.env.container.file('copied-man1')
|
|
file_contents = file_item.read()
|
|
self.assertEqual(
|
|
file_contents,
|
|
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
|
|
# The copied object must not have X-Object-Manifest
|
|
self.assertNotIn("x_object_manifest", file_item.info())
|
|
|
|
def test_copy_manifest(self):
|
|
# Copying the manifest with multipart-manifest=get query string
|
|
# should result in another manifest
|
|
try:
|
|
man1_item = self.env.container.file('man1')
|
|
man1_item.copy(self.env.container.name, "copied-man1",
|
|
parms={'multipart-manifest': 'get'})
|
|
|
|
copied = self.env.container.file("copied-man1")
|
|
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(copied_contents, b"man1-contents")
|
|
|
|
copied_contents = copied.read()
|
|
self.assertEqual(
|
|
copied_contents,
|
|
b"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeee")
|
|
self.assertEqual(man1_item.info()['x_object_manifest'],
|
|
copied.info()['x_object_manifest'])
|
|
finally:
|
|
# try not to leave this around for other tests to stumble over
|
|
self.env.container.file("copied-man1").delete(
|
|
tolerate_missing=True)
|
|
|
|
def test_dlo_if_match_get(self):
|
|
manifest = self.env.container.file("man1")
|
|
etag = manifest.info()['etag']
|
|
|
|
self.assertRaises(ResponseError, manifest.read,
|
|
hdrs={'If-Match': 'not-%s' % etag})
|
|
self.assert_status(412)
|
|
|
|
manifest.read(hdrs={'If-Match': etag})
|
|
self.assert_status(200)
|
|
|
|
def test_dlo_if_none_match_get(self):
|
|
manifest = self.env.container.file("man1")
|
|
etag = manifest.info()['etag']
|
|
|
|
self.assertRaises(ResponseError, manifest.read,
|
|
hdrs={'If-None-Match': etag})
|
|
self.assert_status(304)
|
|
|
|
manifest.read(hdrs={'If-None-Match': "not-%s" % etag})
|
|
self.assert_status(200)
|
|
|
|
def test_dlo_if_match_head(self):
|
|
manifest = self.env.container.file("man1")
|
|
etag = manifest.info()['etag']
|
|
|
|
self.assertRaises(ResponseError, manifest.info,
|
|
hdrs={'If-Match': 'not-%s' % etag})
|
|
self.assert_status(412)
|
|
|
|
manifest.info(hdrs={'If-Match': etag})
|
|
self.assert_status(200)
|
|
|
|
def test_dlo_if_none_match_head(self):
|
|
manifest = self.env.container.file("man1")
|
|
etag = manifest.info()['etag']
|
|
|
|
self.assertRaises(ResponseError, manifest.info,
|
|
hdrs={'If-None-Match': etag})
|
|
self.assert_status(304)
|
|
|
|
manifest.info(hdrs={'If-None-Match': "not-%s" % etag})
|
|
self.assert_status(200)
|
|
|
|
def test_dlo_referer_on_segment_container(self):
|
|
if 'username3' not in tf.config:
|
|
self.skipTest('Requires user 3')
|
|
# First the account2 (test3) should fail
|
|
config2 = tf.config.copy()
|
|
config2['username'] = tf.config['username3']
|
|
config2['password'] = tf.config['password3']
|
|
conn2 = Connection(config2)
|
|
conn2.authenticate()
|
|
headers = {'X-Auth-Token': conn2.storage_token,
|
|
'Referer': 'http://blah.example.com'}
|
|
dlo_file = self.env.container.file("mancont2")
|
|
self.assertRaises(ResponseError, dlo_file.read,
|
|
hdrs=headers)
|
|
self.assert_status(403)
|
|
|
|
# Now set the referer on the dlo container only
|
|
referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'}
|
|
self.env.container.update_metadata(referer_metadata)
|
|
|
|
self.assertRaises(ResponseError, dlo_file.read,
|
|
hdrs=headers)
|
|
self.assert_status(403)
|
|
|
|
# Finally set the referer on the segment container
|
|
self.env.container2.update_metadata(referer_metadata)
|
|
|
|
contents = dlo_file.read(hdrs=headers)
|
|
self.assertEqual(
|
|
contents,
|
|
b"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
|
|
|
|
def test_dlo_post_with_manifest_header(self):
|
|
# verify that performing a POST to a DLO manifest
|
|
# preserves the fact that it is a manifest file.
|
|
# verify that the x-object-manifest header may be updated.
|
|
|
|
# create a new manifest for this test to avoid test coupling.
|
|
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
|
file_item = self.env.container.file(Utils.create_name())
|
|
file_item.write(b'manifest-contents',
|
|
hdrs={"X-Object-Manifest": x_o_m})
|
|
|
|
# sanity checks
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'manifest-contents', manifest_contents)
|
|
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(expected_contents, contents)
|
|
|
|
# POST a modified x-object-manifest value
|
|
new_x_o_m = x_o_m.rstrip('lower') + 'upper'
|
|
file_item.post({'x-object-meta-foo': 'bar',
|
|
'x-object-manifest': new_x_o_m})
|
|
|
|
# verify that x-object-manifest was updated
|
|
file_item.info()
|
|
resp_headers = [(h.lower(), v)
|
|
for h, v in file_item.conn.response.getheaders()]
|
|
self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
|
|
resp_headers)
|
|
self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)
|
|
|
|
# verify that manifest content was not changed
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'manifest-contents', manifest_contents)
|
|
|
|
# verify that updated manifest points to new content
|
|
expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(expected_contents, contents)
|
|
|
|
# Now revert the manifest to point to original segments, including a
|
|
# multipart-manifest=get param just to check that has no effect
|
|
file_item.post({'x-object-manifest': x_o_m},
|
|
parms={'multipart-manifest': 'get'})
|
|
|
|
# verify that x-object-manifest was reverted
|
|
info = file_item.info()
|
|
self.assertIn('x_object_manifest', info)
|
|
self.assertEqual(x_o_m, info['x_object_manifest'])
|
|
|
|
# verify that manifest content was not changed
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'manifest-contents', manifest_contents)
|
|
|
|
# verify that updated manifest points new content
|
|
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(expected_contents, contents)
|
|
|
|
def test_dlo_post_without_manifest_header(self):
|
|
# verify that a POST to a DLO manifest object with no
|
|
# x-object-manifest header will cause the existing x-object-manifest
|
|
# header to be lost
|
|
|
|
# create a new manifest for this test to avoid test coupling.
|
|
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
|
file_item = self.env.container.file(Utils.create_name())
|
|
file_item.write(b'manifest-contents',
|
|
hdrs={"X-Object-Manifest": x_o_m})
|
|
|
|
# sanity checks
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'manifest-contents', manifest_contents)
|
|
expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(expected_contents, contents)
|
|
|
|
# POST with no x-object-manifest header
|
|
file_item.post({})
|
|
|
|
# verify that existing x-object-manifest was removed
|
|
info = file_item.info()
|
|
self.assertNotIn('x_object_manifest', info)
|
|
|
|
# verify that object content was not changed
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'manifest-contents', manifest_contents)
|
|
|
|
# verify that object is no longer a manifest
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(b'manifest-contents', contents)
|
|
|
|
def test_dlo_post_with_manifest_regular_object(self):
|
|
# verify that performing a POST to a regular object
|
|
# with a manifest header will create a DLO.
|
|
|
|
# Put a regular object
|
|
file_item = self.env.container.file(Utils.create_name())
|
|
file_item.write(b'file contents', hdrs={})
|
|
|
|
# sanity checks
|
|
file_contents = file_item.read(parms={})
|
|
self.assertEqual(b'file contents', file_contents)
|
|
|
|
# get the path associated with man1
|
|
x_o_m = self.env.container.file('man1').info()['x_object_manifest']
|
|
|
|
# POST a x-object-manifest value to the regular object
|
|
file_item.post({'x-object-manifest': x_o_m})
|
|
|
|
# verify that the file is now a manifest
|
|
manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
|
|
self.assertEqual(b'file contents', manifest_contents)
|
|
expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
|
|
contents = file_item.read(parms={})
|
|
self.assertEqual(expected_contents, contents)
|
|
file_item.info()
|
|
resp_headers = [(h.lower(), v)
|
|
for h, v in file_item.conn.response.getheaders()]
|
|
self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)
|
|
|
|
|
|
class TestDloUTF8(Base2, TestDlo):
|
|
pass
|