x/swiftonhpss
Code Issues Proposed changes

Sync with OpenStack Swift v1.13.0

Browse Source 
Also, bumped version of gluster-swift to v1.13.0

Change-Id: I797dc704c9523540cba847b1e8ff3da97b79630c
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Reviewed-on: http://review.gluster.org/7229
Reviewed-by: Chetan Risbud <crisbud@redhat.com>
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>
This commit is contained in:
Prashanth Pai 2014-03-12 16:54:30 +05:30 committed by Luis Pabon
parent 1a6b55714f
commit 2ccd2c4d96
12 changed files with 368 additions and 810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doc/markdown/openstack_swift_sync.md
View File

@ -14,7 +14,8 @@ $ python setup.py sdist
$ ls dist
```
* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net
* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net.
* Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad.
## Setup Tox
Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link.
@ -22,7 +23,11 @@ Now that the swift source is availabe on launchpad.net, copy its link location a
## Update tests
This part is a little more complicated and now we need to *merge* the latest tests with ours.
I suggest using a tool called `meld` to make this work easier.
[meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from:
* Latest swift (say v1.13)
* Previous swift (say v1.12)
* gluster-swift (v1.12)
Files that need to be merged:

2
gluster/swift/__init__.py
View File

@ -45,6 +45,6 @@ class PkgInfo(object):
###
### Change the Package version here
###
_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False)
_pkginfo = PkgInfo('1.13.0', '0', 'gluster_swift', False)
__version__ = _pkginfo.pretty_version
__canonical_version__ = _pkginfo.canonical_version

23
test/functional/swift_test_client.py
View File

@ -40,7 +40,7 @@ class RequestError(Exception):
class ResponseError(Exception):
def __init__(self, response, method, path):
def __init__(self, response, method=None, path=None):
self.status = response.status
self.reason = response.reason
self.method = method
@ -310,10 +310,11 @@ class Base:
def __str__(self):
return self.name
def header_fields(self, fields):
def header_fields(self, required_fields, optional_fields=()):
headers = dict(self.conn.response.getheaders())
ret = {}
for field in fields:
for field in required_fields:
if field[1] not in headers:
raise ValueError("%s was not found in response header" %
(field[1]))
@ -322,6 +323,15 @@ class Base:
ret[field[0]] = int(headers[field[1]])
except ValueError:
ret[field[0]] = headers[field[1]]
for field in optional_fields:
if field[1] not in headers:
continue
try:
ret[field[0]] = int(headers[field[1]])
except ValueError:
ret[field[0]] = headers[field[1]]
return ret
@ -480,10 +490,11 @@ class Container(Base):
parms=parms, cfg=cfg)
if self.conn.response.status == 204:
fields = [['bytes_used', 'x-container-bytes-used'],
['object_count', 'x-container-object-count']]
required_fields = [['bytes_used', 'x-container-bytes-used'],
['object_count', 'x-container-object-count']]
optional_fields = [['versions', 'x-versions-location']]
return self.header_fields(fields)
return self.header_fields(required_fields, optional_fields)
raise ResponseError(self.conn.response, 'HEAD',
self.conn.make_path(self.path))

33
test/functional/swift_testing.py
View File

@ -18,6 +18,7 @@ import os
import socket
import sys
from time import sleep
from urlparse import urlparse
from test import get_config
@ -119,18 +120,23 @@ conn = [None, None, None]
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override the 'retries' (default: 5) and
'use_account' (default: 1).
You can use the kwargs to override:
'retries' (default: 5)
'use_account' (default: 1) - which user's token to pass
'url_account' (default: matches 'use_account') - which user's storage URL
'resource' (default: url[url_account] - URL to connect to; retry()
will interpolate the variable :storage_url: if present
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
use_account = 1
if 'use_account' in kwargs:
use_account = kwargs['use_account']
del kwargs['use_account']
use_account -= 1
attempts = 0
backoff = 1
attempts, backoff = 0, 1
# use account #1 by default; turn user's 1-indexed account into 0-indexed
use_account = kwargs.pop('use_account', 1) - 1
# access our own account by default
url_account = kwargs.pop('url_account', use_account + 1) - 1
while attempts <= retries:
attempts += 1
try:
@ -146,8 +152,13 @@ def retry(func, *args, **kwargs):
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account],
# default resource is the account url[url_account]
resource = kwargs.pop('resource', '%(storage_url)s')
template_vars = {'storage_url': url[url_account]}
parsed_result = urlparse(resource % template_vars)
return func(url[url_account], token[use_account],
parsed_result, conn[url_account],
*args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:

148
test/functional/test_account.py
View File

@ -16,12 +16,16 @@
# limitations under the License.
import unittest
import json
from nose import SkipTest
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
from swift.common.middleware.acl import format_acl
from test.functional.swift_test_client import Connection
from test import get_config
from swift_testing import check_response, retry, skip, web_front_end
import swift_testing
class TestAccount(unittest.TestCase):
@ -66,6 +70,148 @@ class TestAccount(unittest.TestCase):
self.assert_(resp.status in (200, 204), resp.status)
self.assertEquals(resp.getheader('x-account-meta-test'), 'Value')
def test_tempauth_account_acls(self):
if skip:
raise SkipTest
# Determine whether this cluster has account ACLs; if not, skip test
conn = Connection(get_config('func_test'))
conn.authenticate()
status = conn.make_request(
'GET', '/info', cfg={'verbatim_path': True})
if status // 100 != 2:
# Can't tell if account ACLs are enabled; skip tests proactively.
raise SkipTest
else:
cluster_info = json.loads(conn.response.read())
if not cluster_info.get('tempauth', {}).get('account_acls'):
raise SkipTest
if 'keystoneauth' in cluster_info:
# Unfortunate hack -- tempauth (with account ACLs) is expected
# to play nice with Keystone (without account ACLs), but Zuul
# functest framework doesn't give us an easy way to get a
# tempauth user.
raise SkipTest
def post(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('POST', parsed.path, '', new_headers)
return check_response(conn)
def put(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path, '', new_headers)
return check_response(conn)
def delete(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('DELETE', parsed.path, '', new_headers)
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
def get(url, token, parsed, conn):
conn.request('GET', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
try:
# User1 can POST to their own account (and reset the ACLs)
resp = retry(post, headers={'X-Account-Access-Control': '{}'},
use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
# User1 can GET their own empty account
resp = retry(get, use_account=1)
resp.read()
self.assertEqual(resp.status // 100, 2)
self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
# User2 can't GET User1's account
resp = retry(get, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 403)
# User1 is swift_owner of their own account, so they can POST an
# ACL -- let's do this and make User2 (test_user[1]) an admin
acl_user = swift_testing.swift_test_user[1]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User1 can see the new header
resp = retry(get, use_account=1)
resp.read()
self.assertEqual(resp.status // 100, 2)
data_from_headers = resp.getheader('x-account-access-control')
expected = json.dumps(acl, separators=(',', ':'))
self.assertEqual(data_from_headers, expected)
# Now User2 should be able to GET the account and see the ACL
resp = retry(head, use_account=2, url_account=1)
resp.read()
data_from_headers = resp.getheader('x-account-access-control')
self.assertEqual(data_from_headers, expected)
# Revoke User2's admin access, grant User2 read-write access
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User2 can still GET the account, but not see the ACL
# (since it's privileged data)
resp = retry(head, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('x-account-access-control'), None)
# User2 can PUT and DELETE a container
resp = retry(put, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(delete, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 204)
# Revoke User2's read-write access, grant User2 read-only access
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User2 can still GET the account, but not see the ACL
# (since it's privileged data)
resp = retry(head, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('x-account-access-control'), None)
# User2 can't PUT a container
resp = retry(put, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 403)
finally:
# Make sure to clean up even if tests fail -- User2 should not
# have access to User1's account in other functional tests!
resp = retry(post, headers={'X-Account-Access-Control': '{}'},
use_account=1)
resp.read()
def test_unicode_metadata(self):
if skip:
raise SkipTest

12
test/functional/tests.py
View File

@ -149,7 +149,7 @@ def timeout(seconds, method, *args, **kwargs):
return False
class Utils:
class Utils(object):
@classmethod
def create_ascii_name(cls, length=None):
return uuid.uuid4().hex
@ -201,7 +201,7 @@ class Base2(object):
Utils.create_name = Utils.create_ascii_name
class TestAccountEnv:
class TestAccountEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@ -388,7 +388,7 @@ class TestAccountUTF8(Base2, TestAccount):
set_up = False
class TestAccountNoContainersEnv:
class TestAccountNoContainersEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@ -417,7 +417,7 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
set_up = False
class TestContainerEnv:
class TestContainerEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)
@ -708,7 +708,7 @@ class TestContainerUTF8(Base2, TestContainer):
set_up = False
class TestContainerPathsEnv:
class TestContainerPathsEnv(object):
@classmethod
def setUp(cls):
raise SkipTest('Objects ending in / are not supported')
@ -888,7 +888,7 @@ class TestContainerPaths(Base):
['dir1/subdir with spaces/file B'])
class TestFileEnv:
class TestFileEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(config)

89
test/unit/proxy/controllers/test_account.py
View File

@ -16,12 +16,14 @@
import mock
import unittest
from swift.common.swob import Request
from swift.common.swob import Request, Response
from swift.common.middleware.acl import format_acl
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base
class TestAccountController(unittest.TestCase):
@ -152,6 +154,91 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def _make_user_and_sys_acl_headers_data(self):
acl = {
'admin': ['AUTH_alice', 'AUTH_bob'],
'read-write': ['AUTH_carol'],
'read-only': [],
}
user_prefix = 'x-account-' # external, user-facing
user_headers = {(user_prefix + 'access-control'): format_acl(
version=2, acl_dict=acl)}
sys_prefix = get_sys_meta_prefix('account') # internal, system-facing
sys_headers = {(sys_prefix + 'core-access-control'): format_acl(
version=2, acl_dict=acl)}
return user_headers, sys_headers
def test_account_acl_headers_translated_for_GET_HEAD(self):
# Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers
# from the account server will remap those headers to X-Account-Acl-*
hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
controller = proxy_server.AccountController(self.app, 'acct')
for verb in ('GET', 'HEAD'):
req = Request.blank('/v1/acct', environ={'swift_owner': True})
controller.GETorHEAD_base = lambda *_: Response(
headers=hdrs_int, environ={
'PATH_INFO': '/acct',
'REQUEST_METHOD': verb,
})
method = getattr(controller, verb)
resp = method(req)
for header, value in hdrs_ext.items():
if value:
self.assertEqual(resp.headers.get(header), value)
else:
# blank ACLs should result in no header
self.assert_(header not in resp.headers)
def test_add_acls_impossible_cases(self):
# For test coverage: verify that defensive coding does defend, in cases
# that shouldn't arise naturally
# add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST
resp = Response()
controller = proxy_server.AccountController(self.app, 'a')
resp.environ['PATH_INFO'] = '/a'
resp.environ['REQUEST_METHOD'] = 'OPTIONS'
controller.add_acls_from_sys_metadata(resp)
self.assertEqual(1, len(resp.headers)) # we always get Content-Type
self.assertEqual(2, len(resp.environ))
def test_memcache_key_impossible_cases(self):
# For test coverage: verify that defensive coding does defend, in cases
# that shouldn't arise naturally
self.assertRaises(
ValueError,
lambda: swift.proxy.controllers.base.get_container_memcache_key(
'/a', None))
def test_stripping_swift_admin_headers(self):
# Verify that a GET/HEAD which receives privileged headers from the
# account server will strip those headers for non-swift_owners
hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
headers = {
'x-account-meta-harmless': 'hi mom',
'x-account-meta-temp-url-key': 's3kr1t',
}
controller = proxy_server.AccountController(self.app, 'acct')
for verb in ('GET', 'HEAD'):
for env in ({'swift_owner': True}, {'swift_owner': False}):
req = Request.blank('/v1/acct', environ=env)
controller.GETorHEAD_base = lambda *_: Response(
headers=headers, environ={
'PATH_INFO': '/acct',
'REQUEST_METHOD': verb,
})
method = getattr(controller, verb)
resp = method(req)
self.assertEqual(resp.headers.get('x-account-meta-harmless'),
'hi mom')
privileged_header_present = (
'x-account-meta-temp-url-key' in resp.headers)
self.assertEqual(privileged_header_present, env['swift_owner'])
if __name__ == '__main__':
unittest.main()

13
test/unit/proxy/controllers/test_obj.py
View File

@ -21,6 +21,7 @@ import mock
import swift
from swift.proxy import server as proxy_server
from swift.common.swob import HTTPException
from test.unit import FakeRing, FakeMemcache, fake_http_connect
@ -44,7 +45,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
self.app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9))
self.app.request_node_count = lambda ring: 10000000
self.app.request_node_count = lambda replicas: 10000000
self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes
def test_iter_nodes_local_first_noops_when_no_affinity(self):
@ -107,14 +108,20 @@ class TestObjController(unittest.TestCase):
# and now test that we add the header to log_info
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.headers['x-copy-from'] = 'somewhere'
controller.PUT(req)
try:
controller.PUT(req)
except HTTPException:
pass
self.assertEquals(
req.environ.get('swift.log_info'), ['x-copy-from:somewhere'])
# and then check that we don't do that for originating POSTs
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.method = 'POST'
req.headers['x-copy-from'] = 'elsewhere'
controller.PUT(req)
try:
controller.PUT(req)
except HTTPException:
pass
self.assertEquals(req.environ.get('swift.log_info'), None)

840
test/unit/proxy/test_server.py
View File

@ -46,7 +46,8 @@ from gluster.swift.container import server as container_server
from gluster.swift.obj import server as object_server
from swift.common import ring
from swift.common.middleware import proxy_logging
from swift.common.exceptions import ChunkReadTimeout, SegmentError
from swift.common.middleware.acl import parse_acl, format_acl
from swift.common.exceptions import ChunkReadTimeout
from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \
@ -54,38 +55,28 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools
from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_memcache_key, cors_validation
import swift.proxy.controllers
from swift.common.swob import Request, Response, HTTPNotFound, \
HTTPUnauthorized
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.swob import Request, Response, HTTPUnauthorized
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
STATIC_TIME = time.time()
_request_instances = weakref.WeakKeyDictionary()
_test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \
_testdir = _orig_SysLogHandler = None
def request_init(self, *args, **kwargs):
self._orig_init(*args, **kwargs)
_request_instances[self] = None
def do_setup(the_object_server):
utils.HASH_PATH_SUFFIX = 'endcap'
global _testdir, _test_servers, _test_sockets, \
_orig_container_listing_limit, _test_coros, _orig_SysLogHandler
_orig_SysLogHandler = utils.SysLogHandler
utils.SysLogHandler = mock.MagicMock()
Request._orig_init = Request.__init__
Request.__init__ = request_init
monkey_patch_mimetools()
# Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of
@ -99,8 +90,6 @@ def do_setup(the_object_server):
mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
mkdirs(os.path.join(_testdir, 'a'))
mkdirs(os.path.join(_testdir, 'a', 'tmp'))
_orig_container_listing_limit = \
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT
conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo',
@ -211,10 +200,7 @@ def setup():
def teardown():
for server in _test_coros:
server.kill()
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \
_orig_container_listing_limit
rmtree(os.path.dirname(_testdir))
Request.__init__ = Request._orig_init
utils.SysLogHandler = _orig_SysLogHandler
@ -1716,7 +1702,7 @@ class TestObjectController(unittest.TestCase):
except ChunkReadTimeout:
got_exc = True
self.assert_(not got_exc)
self.app.node_timeout = 0.1
self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=True)
resp = req.get_response(self.app)
got_exc = False
@ -1731,7 +1717,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req)
self.app.node_timeout = 0.1
self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=[3])
resp = req.get_response(self.app)
got_exc = False
@ -3316,402 +3302,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd)
self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self):
raise SkipTest("Not until we support pure object requests")
# Create a container for our segmented/manifest object testing
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Storage-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object segments
segment_etags = []
for segment in xrange(5):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: '
't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
segment_etags.append(md5('1234 ').hexdigest())
# Create the nonzero size manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: '
't\r\nContent-Length: 5\r\n\r\nabcd ')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n'
'X-Object-Manifest: segmented_nonzero/name/\r\n'
'Foo: barbaz\r\nContent-Type: text/jibberish\r\n'
'\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 202'
self.assertEquals(headers[:len(exp)], exp)
# Ensure retrieving the manifest file gets the whole object
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: '
't\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
self.assert_('Etag: "%s"' % expected_etag in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Get lobjects with Range smaller than manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
'Range: bytes=0-4\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 206'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
body = fd.read()
self.assertEquals(body, '1234 ')
# Get lobjects with Range bigger than manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
'Range: bytes=11-15\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 206'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
body = fd.read()
self.assertEquals(body, '234 1')
def test_chunked_put_lobjects(self):
raise SkipTest("Not until we support pure object requests")
# Create a container for our segmented/manifest object testing
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object segments
segment_etags = []
for segment in xrange(5):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s '
'HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 5\r\n'
'\r\n'
'1234 ' % str(segment))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
segment_etags.append(md5('1234 ').hexdigest())
# Create the object manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Object-Manifest: segmented%20object/object%20name/\r\n'
'Content-Type: text/jibberish\r\n'
'Foo: barbaz\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Check retrieving the listing the manifest would retrieve
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ '
'HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(
body,
'object name/0\n'
'object name/1\n'
'object name/2\n'
'object name/3\n'
'object name/4\n')
# Ensure retrieving the manifest file gets the whole object
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
self.assert_('Etag: "%s"' % expected_etag in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Do it again but exceeding the container listing limit
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
headers)
self.assert_('Content-Type: text/jibberish' in headers)
body = fd.read()
# A bit fragile of a test; as it makes the assumption that all
# will be sent in a single chunk.
self.assertEquals(
body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n')
# Make a copy of the manifested object, which should
# error since the number of segments exceeds
# CONTAINER_LISTING_LIMIT.
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'X-Copy-From: segmented%20object/object%20name\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 413'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
# After adjusting the CONTAINER_LISTING_LIMIT, make a copy of
# the manifested object which should consolidate the segments.
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'X-Copy-From: segmented%20object/object%20name\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
# Retrieve and validate the copy.
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('x-object-manifest:' not in headers.lower())
self.assert_('Content-Length: 25\r' in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Create an object manifest file pointing to nothing
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Object-Manifest: segmented%20object/empty/\r\n'
'Content-Type: text/jibberish\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure retrieving the manifest file gives a zero-byte file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
body = fd.read()
self.assertEquals(body, '')
# Check copy content type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: text/jibberish\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Copy-From: c/obj\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: text/jibberish' in headers)
# Check set content type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: foo/bar\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: foo/bar' in
headers.split('\r\n'), repr(headers.split('\r\n')))
# Check set content type with charset
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: foo/bar; charset=UTF-8\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: foo/bar; charset=UTF-8' in
headers.split('\r\n'), repr(headers.split('\r\n')))
def test_mismatched_etags(self):
with save_globals():
# no etag supplied, object servers return success w/ diff values
@ -4085,53 +3675,63 @@ class TestObjectController(unittest.TestCase):
self.assertTrue('X-Delete-At in past' in resp.body)
def test_leak_1(self):
prolis = _test_sockets[0]
prosrv = _test_servers[0]
obj_len = prosrv.client_chunk_size * 2
# PUT test file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'Content-Length: %s\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n%s' % (obj_len, 'a' * obj_len))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEqual(headers[:len(exp)], exp)
# Remember Request instance count, make sure the GC is run for pythons
# without reference counting.
for i in xrange(4):
sleep(0) # let eventlet do its thing
gc.collect()
else:
sleep(0)
before_request_instances = len(_request_instances)
# GET test file, but disconnect early
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEqual(headers[:len(exp)], exp)
fd.read(1)
fd.close()
sock.close()
# Make sure the GC is run again for pythons without reference counting
for i in xrange(4):
sleep(0) # let eventlet do its thing
gc.collect()
else:
sleep(0)
self.assertEquals(before_request_instances, len(_request_instances))
_request_instances = weakref.WeakKeyDictionary()
_orig_init = Request.__init__
def request_init(self, *args, **kwargs):
_orig_init(self, *args, **kwargs)
_request_instances[self] = None
with mock.patch.object(Request, "__init__", request_init):
prolis = _test_sockets[0]
prosrv = _test_servers[0]
obj_len = prosrv.client_chunk_size * 2
# PUT test file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'Content-Length: %s\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n%s' % (obj_len, 'a' * obj_len))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEqual(headers[:len(exp)], exp)
# Remember Request instance count, make sure the GC is run for
# pythons without reference counting.
for i in xrange(4):
sleep(0) # let eventlet do its thing
gc.collect()
else:
sleep(0)
before_request_instances = len(_request_instances)
# GET test file, but disconnect early
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEqual(headers[:len(exp)], exp)
fd.read(1)
fd.close()
sock.close()
# Make sure the GC is run again for pythons without reference
# counting
for i in xrange(4):
sleep(0) # let eventlet do its thing
gc.collect()
else:
sleep(0)
self.assertEquals(
before_request_instances, len(_request_instances))
def test_OPTIONS(self):
with save_globals():
@ -4926,9 +4526,10 @@ class TestContainerController(unittest.TestCase):
controller = \
proxy_server.ContainerController(self.app, 'a', 'c')
set_http_connect(200, 201, 201, 201, give_connect=test_connect)
req = Request.blank('/v1/a/c',
environ={'REQUEST_METHOD': method},
headers={test_header: test_value})
req = Request.blank(
'/v1/a/c',
environ={'REQUEST_METHOD': method, 'swift_owner': True},
headers={test_header: test_value})
self.app.update_request(req)
getattr(controller, method)(req)
self.assertEquals(test_errors, [])
@ -6006,313 +5607,6 @@ class Stub(object):
pass
class TestSegmentedIterable(unittest.TestCase):
def setUp(self):
self.controller = FakeObjectController()
def test_load_next_segment_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception,
SegmentedIterable(self.controller, None,
[None])._load_next_segment)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_load_next_segment_with_no_segments(self):
self.assertRaises(StopIteration,
SegmentedIterable(self.controller, 'lc',
[])._load_next_segment)
def test_load_next_segment_with_one_segment(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[0][4], '/a/lc/o1')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '1')
def test_load_next_segment_with_two_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o1')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '1')
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '22')
def test_load_next_segment_rate_limiting(self):
sleep_calls = []
def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep
try:
swift.proxy.controllers.obj.sleep = _stub_sleep
segit = SegmentedIterable(
self.controller, 'lc', [
{'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
{'name': 'o4'}, {'name': 'o5'}])
# rate_limit_after_segment == 3, so the first 3 segments should
# invoke no sleeping.
for _ in xrange(3):
segit._load_next_segment()
self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o3')
# Loading of next (4th) segment starts rate-limiting.
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o4')
sleep_calls = []
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o5')
finally:
swift.proxy.controllers.obj.sleep = orig_sleep
def test_load_next_segment_range_req_rate_limiting(self):
sleep_calls = []
def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep
try:
swift.proxy.controllers.obj.sleep = _stub_sleep
segit = SegmentedIterable(
self.controller, 'lc', [
{'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5},
{'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'},
{'name': 'o5'}, {'name': 'o6'}])
# this tests for a range request which skips over the whole first
# segment, after that 3 segments will be read in because the
# rate_limit_after_segment == 3, then sleeping starts
segit_iter = segit.app_iter_range(10, None)
segit_iter.next()
for _ in xrange(2):
# this is set to 2 instead of 3 because o2 was loaded after
# o0 and o1 were skipped.
segit._load_next_segment()
self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o4')
# Loading of next (5th) segment starts rate-limiting.
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o5')
sleep_calls = []
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o6')
finally:
swift.proxy.controllers.obj.sleep = orig_sleep
def test_load_next_segment_with_two_segments_skip_first(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit.ratelimit_index = 0
segit.listing.next()
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '22')
def test_load_next_segment_with_seek(self):
segit = SegmentedIterable(self.controller, 'lc',
[{'name': 'o1', 'bytes': 1},
{'name': 'o2', 'bytes': 2}])
segit.ratelimit_index = 0
segit.listing.next()
segit.seek = 1
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
self.assertEquals(
str(self.controller.GETorHEAD_base_args[-1][0].range),
'bytes=1-')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '2')
def test_fetching_only_what_you_need(self):
segit = SegmentedIterable(self.controller, 'lc',
[{'name': 'o7', 'bytes': 7},
{'name': 'o8', 'bytes': 8},
{'name': 'o9', 'bytes': 9}])
body = ''.join(segit.app_iter_range(10, 20))
self.assertEqual('8888899999', body)
GoH_args = self.controller.GETorHEAD_base_args
self.assertEquals(2, len(GoH_args))
# Either one is fine, as they both indicate "from byte 3 to (the last)
# byte 8".
self.assert_(str(GoH_args[0][0].range) in ['bytes=3-', 'bytes=3-8'])
# This one must ask only for the bytes it needs; otherwise we waste
# bandwidth pulling bytes from the object server and then throwing
# them out
self.assertEquals(str(GoH_args[1][0].range), 'bytes=0-4')
def test_load_next_segment_with_get_error(self):
def local_GETorHEAD_base(*args):
return HTTPNotFound()
self.controller.GETorHEAD_base = local_GETorHEAD_base
self.assertRaises(Exception,
SegmentedIterable(self.controller, 'lc',
[{'name': 'o1'}])._load_next_segment)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404')
def test_iter_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception, ''.join,
SegmentedIterable(self.controller, None, [None]))
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_iter_with_no_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [])
self.assertEquals(''.join(segit), '')
def test_iter_with_one_segment(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])
segit.response = Stub()
self.assertEquals(''.join(segit), '1')
def test_iter_with_two_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit.response = Stub()
self.assertEquals(''.join(segit), '122')
def test_iter_with_get_error(self):
def local_GETorHEAD_base(*args):
return HTTPNotFound()
self.controller.GETorHEAD_base = local_GETorHEAD_base
self.assertRaises(Exception, ''.join,
SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}]))
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404')
def test_app_iter_range_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception,
SegmentedIterable(self.controller, None,
[None]).app_iter_range(None,
None).next)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_app_iter_range_with_no_segments(self):
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(None, None)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(3, None)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(3, 5)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(None, 5)), '')
def test_app_iter_range_with_one_segment(self):
listing = [{'name': 'o1', 'bytes': 1}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)), '1')
segit = SegmentedIterable(self.controller, 'lc', listing)
self.assertEquals(''.join(segit.app_iter_range(3, None)), '')
segit = SegmentedIterable(self.controller, 'lc', listing)
self.assertEquals(''.join(segit.app_iter_range(3, 5)), '')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1')
def test_app_iter_range_with_two_segments(self):
listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)), '122')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(1, None)), '22')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12')
def test_app_iter_range_with_many_segments(self):
listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2},
{'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4},
{'name': 'o5', 'bytes': 5}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)),
'122333444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(3, None)),
'333444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
class TestProxyObjectPerformance(unittest.TestCase):
def setUp(self):

1
tools/requirements.txt
View File

@ -5,4 +5,3 @@ netifaces>=0.5
pastedeploy>=1.3.3
simplejson>=2.0.9
xattr>=0.4
python-swiftclient

6
tools/test-requires
View File

@ -1,7 +1,4 @@
# Install bounded pep8/pyflakes first, then let flake8 install
pep8==1.4.5
pyflakes==0.7.2
flake8==2.0
# Hacking already pins down pep8, pyflakes and flake8
hacking>=0.5.6,<0.6
coverage
nose
@ -10,5 +7,6 @@ openstack.nose_plugin
nosehtmloutput
sphinx>=1.1.2,<1.2
mock>=0.8.0
python-swiftclient
python-keystoneclient
prettytable

2
tox.ini
View File

@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps =
https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz
https://launchpad.net/gluster-swift/icehouse/1.13.0/+download/swift-1.13.0.tar.gz
--download-cache={homedir}/.pipcache
-r{toxinidir}/tools/test-requires
-r{toxinidir}/tools/requirements.txt