x/swiftonhpss
Code Issues Proposed changes

Sync with OpenStack Swift v1.13.0

Browse Source 
Also, bumped version of gluster-swift to v1.13.0

Change-Id: I797dc704c9523540cba847b1e8ff3da97b79630c
Signed-off-by: Prashanth Pai <ppai@redhat.com>
Reviewed-on: http://review.gluster.org/7229
Reviewed-by: Chetan Risbud <crisbud@redhat.com>
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>
This commit is contained in:
Prashanth Pai 2014-03-12 16:54:30 +05:30 committed by Luis Pabon
parent 1a6b55714f
commit 2ccd2c4d96
12 changed files with 368 additions and 810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doc/markdown/openstack_swift_sync.md
View File

@ -14,7 +14,8 @@ $ python setup.py sdist
$ ls dist $ ls dist
``` ```
* Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net * Take the file in the `dist` directory and upload it to the new release we created it on launchpad.net.
* Alternatively, if we are syncing with a Swift version which is already released, we can get the tar.gz file from Swift launchpad page and upload the same to gluster-swift launchpad.
## Setup Tox ## Setup Tox
Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link. Now that the swift source is availabe on launchpad.net, copy its link location and update tox.ini in gluster-swift with the new link.
@ -22,7 +23,11 @@ Now that the swift source is availabe on launchpad.net, copy its link location a
## Update tests ## Update tests
This part is a little more complicated and now we need to *merge* the latest tests with ours. This part is a little more complicated and now we need to *merge* the latest tests with ours.
I suggest using a tool called `meld` to make this work easier. [meld](http://meldmerge.org/) is a great tool to make this work easier. The 3-way comparison feature of meld comes handy to compare 3 version of same file from:
* Latest swift (say v1.13)
* Previous swift (say v1.12)
* gluster-swift (v1.12)
Files that need to be merged: Files that need to be merged:

2
gluster/swift/__init__.py
View File

@ -45,6 +45,6 @@ class PkgInfo(object):
### ###
### Change the Package version here ### Change the Package version here
### ###
_pkginfo = PkgInfo('1.12.0', '0', 'gluster_swift', False) _pkginfo = PkgInfo('1.13.0', '0', 'gluster_swift', False)
__version__ = _pkginfo.pretty_version __version__ = _pkginfo.pretty_version
__canonical_version__ = _pkginfo.canonical_version __canonical_version__ = _pkginfo.canonical_version

23
test/functional/swift_test_client.py
View File

@ -40,7 +40,7 @@ class RequestError(Exception):
class ResponseError(Exception): class ResponseError(Exception):
def __init__(self, response, method, path): def __init__(self, response, method=None, path=None):
self.status = response.status self.status = response.status
self.reason = response.reason self.reason = response.reason
self.method = method self.method = method
@ -310,10 +310,11 @@ class Base:
def __str__(self): def __str__(self):
return self.name return self.name
def header_fields(self, fields): def header_fields(self, required_fields, optional_fields=()):
headers = dict(self.conn.response.getheaders()) headers = dict(self.conn.response.getheaders())
ret = {} ret = {}
for field in fields:
for field in required_fields:
if field[1] not in headers: if field[1] not in headers:
raise ValueError("%s was not found in response header" % raise ValueError("%s was not found in response header" %
(field[1])) (field[1]))
@ -322,6 +323,15 @@ class Base:
ret[field[0]] = int(headers[field[1]]) ret[field[0]] = int(headers[field[1]])
except ValueError: except ValueError:
ret[field[0]] = headers[field[1]] ret[field[0]] = headers[field[1]]
for field in optional_fields:
if field[1] not in headers:
continue
try:
ret[field[0]] = int(headers[field[1]])
except ValueError:
ret[field[0]] = headers[field[1]]
return ret return ret
@ -480,10 +490,11 @@ class Container(Base):
parms=parms, cfg=cfg) parms=parms, cfg=cfg)
if self.conn.response.status == 204: if self.conn.response.status == 204:
fields = [['bytes_used', 'x-container-bytes-used'], required_fields = [['bytes_used', 'x-container-bytes-used'],
['object_count', 'x-container-object-count']] ['object_count', 'x-container-object-count']]
optional_fields = [['versions', 'x-versions-location']]
return self.header_fields(fields) return self.header_fields(required_fields, optional_fields)
raise ResponseError(self.conn.response, 'HEAD', raise ResponseError(self.conn.response, 'HEAD',
self.conn.make_path(self.path)) self.conn.make_path(self.path))

33
test/functional/swift_testing.py
View File

@ -18,6 +18,7 @@ import os
import socket import socket
import sys import sys
from time import sleep from time import sleep
from urlparse import urlparse
from test import get_config from test import get_config
@ -119,18 +120,23 @@ conn = [None, None, None]
def retry(func, *args, **kwargs): def retry(func, *args, **kwargs):
""" """
You can use the kwargs to override the 'retries' (default: 5) and You can use the kwargs to override:
'use_account' (default: 1). 'retries' (default: 5)
'use_account' (default: 1) - which user's token to pass
'url_account' (default: matches 'use_account') - which user's storage URL
'resource' (default: url[url_account] - URL to connect to; retry()
will interpolate the variable :storage_url: if present
""" """
global url, token, parsed, conn global url, token, parsed, conn
retries = kwargs.get('retries', 5) retries = kwargs.get('retries', 5)
use_account = 1 attempts, backoff = 0, 1
if 'use_account' in kwargs:
use_account = kwargs['use_account'] # use account #1 by default; turn user's 1-indexed account into 0-indexed
del kwargs['use_account'] use_account = kwargs.pop('use_account', 1) - 1
use_account -= 1
attempts = 0 # access our own account by default
backoff = 1 url_account = kwargs.pop('url_account', use_account + 1) - 1
while attempts <= retries: while attempts <= retries:
attempts += 1 attempts += 1
try: try:
@ -146,8 +152,13 @@ def retry(func, *args, **kwargs):
if not parsed[use_account] or not conn[use_account]: if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \ parsed[use_account], conn[use_account] = \
http_connection(url[use_account]) http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account], # default resource is the account url[url_account]
resource = kwargs.pop('resource', '%(storage_url)s')
template_vars = {'storage_url': url[url_account]}
parsed_result = urlparse(resource % template_vars)
return func(url[url_account], token[use_account],
parsed_result, conn[url_account],
*args, **kwargs) *args, **kwargs)
except (socket.error, HTTPException): except (socket.error, HTTPException):
if attempts > retries: if attempts > retries:

148
test/functional/test_account.py
View File

@ -16,12 +16,16 @@
# limitations under the License. # limitations under the License.
import unittest import unittest
import json
from nose import SkipTest from nose import SkipTest
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
from swift.common.middleware.acl import format_acl
from test.functional.swift_test_client import Connection
from test import get_config
from swift_testing import check_response, retry, skip, web_front_end from swift_testing import check_response, retry, skip, web_front_end
import swift_testing
class TestAccount(unittest.TestCase): class TestAccount(unittest.TestCase):
@ -66,6 +70,148 @@ class TestAccount(unittest.TestCase):
self.assert_(resp.status in (200, 204), resp.status) self.assert_(resp.status in (200, 204), resp.status)
self.assertEquals(resp.getheader('x-account-meta-test'), 'Value') self.assertEquals(resp.getheader('x-account-meta-test'), 'Value')
def test_tempauth_account_acls(self):
if skip:
raise SkipTest
# Determine whether this cluster has account ACLs; if not, skip test
conn = Connection(get_config('func_test'))
conn.authenticate()
status = conn.make_request(
'GET', '/info', cfg={'verbatim_path': True})
if status // 100 != 2:
# Can't tell if account ACLs are enabled; skip tests proactively.
raise SkipTest
else:
cluster_info = json.loads(conn.response.read())
if not cluster_info.get('tempauth', {}).get('account_acls'):
raise SkipTest
if 'keystoneauth' in cluster_info:
# Unfortunate hack -- tempauth (with account ACLs) is expected
# to play nice with Keystone (without account ACLs), but Zuul
# functest framework doesn't give us an easy way to get a
# tempauth user.
raise SkipTest
def post(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('POST', parsed.path, '', new_headers)
return check_response(conn)
def put(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('PUT', parsed.path, '', new_headers)
return check_response(conn)
def delete(url, token, parsed, conn, headers):
new_headers = dict({'X-Auth-Token': token}, **headers)
conn.request('DELETE', parsed.path, '', new_headers)
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
def get(url, token, parsed, conn):
conn.request('GET', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
try:
# User1 can POST to their own account (and reset the ACLs)
resp = retry(post, headers={'X-Account-Access-Control': '{}'},
use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
# User1 can GET their own empty account
resp = retry(get, use_account=1)
resp.read()
self.assertEqual(resp.status // 100, 2)
self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
# User2 can't GET User1's account
resp = retry(get, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 403)
# User1 is swift_owner of their own account, so they can POST an
# ACL -- let's do this and make User2 (test_user[1]) an admin
acl_user = swift_testing.swift_test_user[1]
acl = {'admin': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User1 can see the new header
resp = retry(get, use_account=1)
resp.read()
self.assertEqual(resp.status // 100, 2)
data_from_headers = resp.getheader('x-account-access-control')
expected = json.dumps(acl, separators=(',', ':'))
self.assertEqual(data_from_headers, expected)
# Now User2 should be able to GET the account and see the ACL
resp = retry(head, use_account=2, url_account=1)
resp.read()
data_from_headers = resp.getheader('x-account-access-control')
self.assertEqual(data_from_headers, expected)
# Revoke User2's admin access, grant User2 read-write access
acl = {'read-write': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User2 can still GET the account, but not see the ACL
# (since it's privileged data)
resp = retry(head, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('x-account-access-control'), None)
# User2 can PUT and DELETE a container
resp = retry(put, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 201)
resp = retry(delete, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 204)
# Revoke User2's read-write access, grant User2 read-only access
acl = {'read-only': [acl_user]}
headers = {'x-account-access-control': format_acl(
version=2, acl_dict=acl)}
resp = retry(post, headers=headers, use_account=1)
resp.read()
self.assertEqual(resp.status, 204)
# User2 can still GET the account, but not see the ACL
# (since it's privileged data)
resp = retry(head, use_account=2, url_account=1)
resp.read()
self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('x-account-access-control'), None)
# User2 can't PUT a container
resp = retry(put, use_account=2, url_account=1,
resource='%(storage_url)s/mycontainer', headers={})
resp.read()
self.assertEqual(resp.status, 403)
finally:
# Make sure to clean up even if tests fail -- User2 should not
# have access to User1's account in other functional tests!
resp = retry(post, headers={'X-Account-Access-Control': '{}'},
use_account=1)
resp.read()
def test_unicode_metadata(self): def test_unicode_metadata(self):
if skip: if skip:
raise SkipTest raise SkipTest

12
test/functional/tests.py
View File

@ -149,7 +149,7 @@ def timeout(seconds, method, *args, **kwargs):
return False return False
class Utils: class Utils(object):
@classmethod @classmethod
def create_ascii_name(cls, length=None): def create_ascii_name(cls, length=None):
return uuid.uuid4().hex return uuid.uuid4().hex
@ -201,7 +201,7 @@ class Base2(object):
Utils.create_name = Utils.create_ascii_name Utils.create_name = Utils.create_ascii_name
class TestAccountEnv: class TestAccountEnv(object):
@classmethod @classmethod
def setUp(cls): def setUp(cls):
cls.conn = Connection(config) cls.conn = Connection(config)
@ -388,7 +388,7 @@ class TestAccountUTF8(Base2, TestAccount):
set_up = False set_up = False
class TestAccountNoContainersEnv: class TestAccountNoContainersEnv(object):
@classmethod @classmethod
def setUp(cls): def setUp(cls):
cls.conn = Connection(config) cls.conn = Connection(config)
@ -417,7 +417,7 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
set_up = False set_up = False
class TestContainerEnv: class TestContainerEnv(object):
@classmethod @classmethod
def setUp(cls): def setUp(cls):
cls.conn = Connection(config) cls.conn = Connection(config)
@ -708,7 +708,7 @@ class TestContainerUTF8(Base2, TestContainer):
set_up = False set_up = False
class TestContainerPathsEnv: class TestContainerPathsEnv(object):
@classmethod @classmethod
def setUp(cls): def setUp(cls):
raise SkipTest('Objects ending in / are not supported') raise SkipTest('Objects ending in / are not supported')
@ -888,7 +888,7 @@ class TestContainerPaths(Base):
['dir1/subdir with spaces/file B']) ['dir1/subdir with spaces/file B'])
class TestFileEnv: class TestFileEnv(object):
@classmethod @classmethod
def setUp(cls): def setUp(cls):
cls.conn = Connection(config) cls.conn = Connection(config)

89
test/unit/proxy/controllers/test_account.py
View File

@ -16,12 +16,14 @@
import mock import mock
import unittest import unittest
from swift.common.swob import Request from swift.common.swob import Request, Response
from swift.common.middleware.acl import format_acl
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info from swift.proxy.controllers.base import headers_to_account_info
from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN from swift.common.constraints import MAX_ACCOUNT_NAME_LENGTH as MAX_ANAME_LEN
from test.unit import fake_http_connect, FakeRing, FakeMemcache from test.unit import fake_http_connect, FakeRing, FakeMemcache
from swift.common.request_helpers import get_sys_meta_prefix from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base
class TestAccountController(unittest.TestCase): class TestAccountController(unittest.TestCase):
@ -152,6 +154,91 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(context['headers'][user_meta_key], 'bar') self.assertEqual(context['headers'][user_meta_key], 'bar')
self.assertNotEqual(context['headers']['x-timestamp'], '1.0') self.assertNotEqual(context['headers']['x-timestamp'], '1.0')
def _make_user_and_sys_acl_headers_data(self):
acl = {
'admin': ['AUTH_alice', 'AUTH_bob'],
'read-write': ['AUTH_carol'],
'read-only': [],
}
user_prefix = 'x-account-' # external, user-facing
user_headers = {(user_prefix + 'access-control'): format_acl(
version=2, acl_dict=acl)}
sys_prefix = get_sys_meta_prefix('account') # internal, system-facing
sys_headers = {(sys_prefix + 'core-access-control'): format_acl(
version=2, acl_dict=acl)}
return user_headers, sys_headers
def test_account_acl_headers_translated_for_GET_HEAD(self):
# Verify that a GET/HEAD which receives X-Account-Sysmeta-Acl-* headers
# from the account server will remap those headers to X-Account-Acl-*
hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
controller = proxy_server.AccountController(self.app, 'acct')
for verb in ('GET', 'HEAD'):
req = Request.blank('/v1/acct', environ={'swift_owner': True})
controller.GETorHEAD_base = lambda *_: Response(
headers=hdrs_int, environ={
'PATH_INFO': '/acct',
'REQUEST_METHOD': verb,
})
method = getattr(controller, verb)
resp = method(req)
for header, value in hdrs_ext.items():
if value:
self.assertEqual(resp.headers.get(header), value)
else:
# blank ACLs should result in no header
self.assert_(header not in resp.headers)
def test_add_acls_impossible_cases(self):
# For test coverage: verify that defensive coding does defend, in cases
# that shouldn't arise naturally
# add_acls should do nothing if REQUEST_METHOD isn't HEAD/GET/PUT/POST
resp = Response()
controller = proxy_server.AccountController(self.app, 'a')
resp.environ['PATH_INFO'] = '/a'
resp.environ['REQUEST_METHOD'] = 'OPTIONS'
controller.add_acls_from_sys_metadata(resp)
self.assertEqual(1, len(resp.headers)) # we always get Content-Type
self.assertEqual(2, len(resp.environ))
def test_memcache_key_impossible_cases(self):
# For test coverage: verify that defensive coding does defend, in cases
# that shouldn't arise naturally
self.assertRaises(
ValueError,
lambda: swift.proxy.controllers.base.get_container_memcache_key(
'/a', None))
def test_stripping_swift_admin_headers(self):
# Verify that a GET/HEAD which receives privileged headers from the
# account server will strip those headers for non-swift_owners
hdrs_ext, hdrs_int = self._make_user_and_sys_acl_headers_data()
headers = {
'x-account-meta-harmless': 'hi mom',
'x-account-meta-temp-url-key': 's3kr1t',
}
controller = proxy_server.AccountController(self.app, 'acct')
for verb in ('GET', 'HEAD'):
for env in ({'swift_owner': True}, {'swift_owner': False}):
req = Request.blank('/v1/acct', environ=env)
controller.GETorHEAD_base = lambda *_: Response(
headers=headers, environ={
'PATH_INFO': '/acct',
'REQUEST_METHOD': verb,
})
method = getattr(controller, verb)
resp = method(req)
self.assertEqual(resp.headers.get('x-account-meta-harmless'),
'hi mom')
privileged_header_present = (
'x-account-meta-temp-url-key' in resp.headers)
self.assertEqual(privileged_header_present, env['swift_owner'])
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

13
test/unit/proxy/controllers/test_obj.py
View File

@ -21,6 +21,7 @@ import mock
import swift import swift
from swift.proxy import server as proxy_server from swift.proxy import server as proxy_server
from swift.common.swob import HTTPException
from test.unit import FakeRing, FakeMemcache, fake_http_connect from test.unit import FakeRing, FakeMemcache, fake_http_connect
@ -44,7 +45,7 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
self.app = proxy_server.Application( self.app = proxy_server.Application(
None, FakeMemcache(), account_ring=FakeRing(), None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9)) container_ring=FakeRing(), object_ring=FakeRing(max_more_nodes=9))
self.app.request_node_count = lambda ring: 10000000 self.app.request_node_count = lambda replicas: 10000000
self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes self.app.sort_nodes = lambda l: l # stop shuffling the primary nodes
def test_iter_nodes_local_first_noops_when_no_affinity(self): def test_iter_nodes_local_first_noops_when_no_affinity(self):
@ -107,14 +108,20 @@ class TestObjController(unittest.TestCase):
# and now test that we add the header to log_info # and now test that we add the header to log_info
req = swift.common.swob.Request.blank('/v1/a/c/o') req = swift.common.swob.Request.blank('/v1/a/c/o')
req.headers['x-copy-from'] = 'somewhere' req.headers['x-copy-from'] = 'somewhere'
controller.PUT(req) try:
controller.PUT(req)
except HTTPException:
pass
self.assertEquals( self.assertEquals(
req.environ.get('swift.log_info'), ['x-copy-from:somewhere']) req.environ.get('swift.log_info'), ['x-copy-from:somewhere'])
# and then check that we don't do that for originating POSTs # and then check that we don't do that for originating POSTs
req = swift.common.swob.Request.blank('/v1/a/c/o') req = swift.common.swob.Request.blank('/v1/a/c/o')
req.method = 'POST' req.method = 'POST'
req.headers['x-copy-from'] = 'elsewhere' req.headers['x-copy-from'] = 'elsewhere'
controller.PUT(req) try:
controller.PUT(req)
except HTTPException:
pass
self.assertEquals(req.environ.get('swift.log_info'), None) self.assertEquals(req.environ.get('swift.log_info'), None)

840
test/unit/proxy/test_server.py
View File

@ -46,7 +46,8 @@ from gluster.swift.container import server as container_server
from gluster.swift.obj import server as object_server from gluster.swift.obj import server as object_server
from swift.common import ring from swift.common import ring
from swift.common.middleware import proxy_logging from swift.common.middleware import proxy_logging
from swift.common.exceptions import ChunkReadTimeout, SegmentError from swift.common.middleware.acl import parse_acl, format_acl
from swift.common.exceptions import ChunkReadTimeout
from swift.common.constraints import MAX_META_NAME_LENGTH, \ from swift.common.constraints import MAX_META_NAME_LENGTH, \
MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \ MAX_META_VALUE_LENGTH, MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \ MAX_FILE_SIZE, MAX_ACCOUNT_NAME_LENGTH, MAX_CONTAINER_NAME_LENGTH, \
@ -54,38 +55,28 @@ from swift.common.constraints import MAX_META_NAME_LENGTH, \
from swift.common import utils from swift.common import utils
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
from swift.common.wsgi import monkey_patch_mimetools from swift.common.wsgi import monkey_patch_mimetools
from swift.proxy.controllers.obj import SegmentedIterable
from swift.proxy.controllers import base as proxy_base from swift.proxy.controllers import base as proxy_base
from swift.proxy.controllers.base import get_container_memcache_key, \ from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_memcache_key, cors_validation get_account_memcache_key, cors_validation
import swift.proxy.controllers import swift.proxy.controllers
from swift.common.swob import Request, Response, HTTPNotFound, \ from swift.common.request_helpers import get_sys_meta_prefix
HTTPUnauthorized from swift.common.swob import Request, Response, HTTPUnauthorized
# mocks # mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
STATIC_TIME = time.time() STATIC_TIME = time.time()
_request_instances = weakref.WeakKeyDictionary()
_test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \ _test_coros = _test_servers = _test_sockets = _orig_container_listing_limit = \
_testdir = _orig_SysLogHandler = None _testdir = _orig_SysLogHandler = None
def request_init(self, *args, **kwargs):
self._orig_init(*args, **kwargs)
_request_instances[self] = None
def do_setup(the_object_server): def do_setup(the_object_server):
utils.HASH_PATH_SUFFIX = 'endcap' utils.HASH_PATH_SUFFIX = 'endcap'
global _testdir, _test_servers, _test_sockets, \ global _testdir, _test_servers, _test_sockets, \
_orig_container_listing_limit, _test_coros, _orig_SysLogHandler _orig_container_listing_limit, _test_coros, _orig_SysLogHandler
_orig_SysLogHandler = utils.SysLogHandler _orig_SysLogHandler = utils.SysLogHandler
utils.SysLogHandler = mock.MagicMock() utils.SysLogHandler = mock.MagicMock()
Request._orig_init = Request.__init__
Request.__init__ = request_init
monkey_patch_mimetools() monkey_patch_mimetools()
# Since we're starting up a lot here, we're going to test more than # Since we're starting up a lot here, we're going to test more than
# just chunked puts; we're also going to test parts of # just chunked puts; we're also going to test parts of
@ -99,8 +90,6 @@ def do_setup(the_object_server):
mkdirs(os.path.join(_testdir, 'sdb1', 'tmp')) mkdirs(os.path.join(_testdir, 'sdb1', 'tmp'))
mkdirs(os.path.join(_testdir, 'a')) mkdirs(os.path.join(_testdir, 'a'))
mkdirs(os.path.join(_testdir, 'a', 'tmp')) mkdirs(os.path.join(_testdir, 'a', 'tmp'))
_orig_container_listing_limit = \
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT
conf = {'devices': _testdir, 'swift_dir': _testdir, conf = {'devices': _testdir, 'swift_dir': _testdir,
'mount_check': 'false', 'allowed_headers': 'mount_check': 'false', 'allowed_headers':
'content-encoding, x-object-manifest, content-disposition, foo', 'content-encoding, x-object-manifest, content-disposition, foo',
@ -211,10 +200,7 @@ def setup():
def teardown(): def teardown():
for server in _test_coros: for server in _test_coros:
server.kill() server.kill()
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = \
_orig_container_listing_limit
rmtree(os.path.dirname(_testdir)) rmtree(os.path.dirname(_testdir))
Request.__init__ = Request._orig_init
utils.SysLogHandler = _orig_SysLogHandler utils.SysLogHandler = _orig_SysLogHandler
@ -1716,7 +1702,7 @@ class TestObjectController(unittest.TestCase):
except ChunkReadTimeout: except ChunkReadTimeout:
got_exc = True got_exc = True
self.assert_(not got_exc) self.assert_(not got_exc)
self.app.node_timeout = 0.1 self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=True) set_http_connect(200, 200, 200, slow=True)
resp = req.get_response(self.app) resp = req.get_response(self.app)
got_exc = False got_exc = False
@ -1731,7 +1717,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'}) req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.app.update_request(req) self.app.update_request(req)
self.app.node_timeout = 0.1 self.app.recoverable_node_timeout = 0.1
set_http_connect(200, 200, 200, slow=[3]) set_http_connect(200, 200, 200, slow=[3])
resp = req.get_response(self.app) resp = req.get_response(self.app)
got_exc = False got_exc = False
@ -3316,402 +3302,6 @@ class TestObjectController(unittest.TestCase):
headers = readuntil2crlfs(fd) headers = readuntil2crlfs(fd)
self.assertEquals(headers[:len(exp)], exp) self.assertEquals(headers[:len(exp)], exp)
def test_chunked_put_lobjects_with_nonzero_size_manifest_file(self):
raise SkipTest("Not until we support pure object requests")
# Create a container for our segmented/manifest object testing
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis, obj2lis) = \
_test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero HTTP/1.1\r\nHost: localhost\r\n'
'Connection: close\r\nX-Storage-Token: t\r\n'
'Content-Length: 0\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object segments
segment_etags = []
for segment in xrange(5):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero/name/%s HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: '
't\r\nContent-Length: 5\r\n\r\n1234 ' % str(segment))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
segment_etags.append(md5('1234 ').hexdigest())
# Create the nonzero size manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: '
't\r\nContent-Length: 5\r\n\r\nabcd ')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('POST /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Storage-Token: t\r\n'
'X-Object-Manifest: segmented_nonzero/name/\r\n'
'Foo: barbaz\r\nContent-Type: text/jibberish\r\n'
'\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 202'
self.assertEquals(headers[:len(exp)], exp)
# Ensure retrieving the manifest file gets the whole object
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: '
't\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
self.assert_('Etag: "%s"' % expected_etag in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Get lobjects with Range smaller than manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
'Range: bytes=0-4\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 206'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
body = fd.read()
self.assertEquals(body, '1234 ')
# Get lobjects with Range bigger than manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented_nonzero/name HTTP/1.1\r\nHost: '
'localhost\r\nConnection: close\r\nX-Auth-Token: t\r\n'
'Range: bytes=11-15\r\n\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 206'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented_nonzero/name/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
body = fd.read()
self.assertEquals(body, '234 1')
def test_chunked_put_lobjects(self):
raise SkipTest("Not until we support pure object requests")
# Create a container for our segmented/manifest object testing
(prolis, acc1lis, acc2lis, con1lis, con2lis, obj1lis,
obj2lis) = _test_sockets
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Create the object segments
segment_etags = []
for segment in xrange(5):
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%%20object/object%%20name/%s '
'HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 5\r\n'
'\r\n'
'1234 ' % str(segment))
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
segment_etags.append(md5('1234 ').hexdigest())
# Create the object manifest file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Object-Manifest: segmented%20object/object%20name/\r\n'
'Content-Type: text/jibberish\r\n'
'Foo: barbaz\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Check retrieving the listing the manifest would retrieve
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object?prefix=object%20name/ '
'HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
self.assertEquals(
body,
'object name/0\n'
'object name/1\n'
'object name/2\n'
'object name/3\n'
'object name/4\n')
# Ensure retrieving the manifest file gets the whole object
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
headers)
self.assert_('Content-Type: text/jibberish' in headers)
self.assert_('Foo: barbaz' in headers)
expected_etag = md5(''.join(segment_etags)).hexdigest()
self.assert_('Etag: "%s"' % expected_etag in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Do it again but exceeding the container listing limit
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 2
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/object%20name HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/object%20name/' in
headers)
self.assert_('Content-Type: text/jibberish' in headers)
body = fd.read()
# A bit fragile of a test; as it makes the assumption that all
# will be sent in a single chunk.
self.assertEquals(
body, '19\r\n1234 1234 1234 1234 1234 \r\n0\r\n\r\n')
# Make a copy of the manifested object, which should
# error since the number of segments exceeds
# CONTAINER_LISTING_LIMIT.
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'X-Copy-From: segmented%20object/object%20name\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 413'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
# After adjusting the CONTAINER_LISTING_LIMIT, make a copy of
# the manifested object which should consolidate the segments.
swift.proxy.controllers.obj.CONTAINER_LISTING_LIMIT = 10000
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'X-Copy-From: segmented%20object/object%20name\r\n'
'Content-Length: 0\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
body = fd.read()
# Retrieve and validate the copy.
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/copy HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('x-object-manifest:' not in headers.lower())
self.assert_('Content-Length: 25\r' in headers)
body = fd.read()
self.assertEquals(body, '1234 1234 1234 1234 1234 ')
# Create an object manifest file pointing to nothing
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/segmented%20object/empty HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Object-Manifest: segmented%20object/empty/\r\n'
'Content-Type: text/jibberish\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure retrieving the manifest file gives a zero-byte file
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/segmented%20object/empty HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('X-Object-Manifest: segmented%20object/empty/' in headers)
self.assert_('Content-Type: text/jibberish' in headers)
body = fd.read()
self.assertEquals(body, '')
# Check copy content type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: text/jibberish\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj2 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'X-Copy-From: c/obj\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj2 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: text/jibberish' in headers)
# Check set content type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj3 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: foo/bar\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj3 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: foo/bar' in
headers.split('\r\n'), repr(headers.split('\r\n')))
# Check set content type with charset
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('PUT /v1/a/c/obj4 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Storage-Token: t\r\n'
'Content-Length: 0\r\n'
'Content-Type: foo/bar; charset=UTF-8\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 201'
self.assertEquals(headers[:len(exp)], exp)
# Ensure getting the copied file gets original content-type
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd = sock.makefile()
fd.write('GET /v1/a/c/obj4 HTTP/1.1\r\n'
'Host: localhost\r\n'
'Connection: close\r\n'
'X-Auth-Token: t\r\n'
'\r\n')
fd.flush()
headers = readuntil2crlfs(fd)
exp = 'HTTP/1.1 200'
self.assertEquals(headers[:len(exp)], exp)
self.assert_('Content-Type: foo/bar; charset=UTF-8' in
headers.split('\r\n'), repr(headers.split('\r\n')))
def test_mismatched_etags(self): def test_mismatched_etags(self):
with save_globals(): with save_globals():
# no etag supplied, object servers return success w/ diff values # no etag supplied, object servers return success w/ diff values
@ -4085,53 +3675,63 @@ class TestObjectController(unittest.TestCase):
self.assertTrue('X-Delete-At in past' in resp.body) self.assertTrue('X-Delete-At in past' in resp.body)
def test_leak_1(self): def test_leak_1(self):
prolis = _test_sockets[0] _request_instances = weakref.WeakKeyDictionary()
prosrv = _test_servers[0] _orig_init = Request.__init__
obj_len = prosrv.client_chunk_size * 2
# PUT test file def request_init(self, *args, **kwargs):
sock = connect_tcp(('localhost', prolis.getsockname()[1])) _orig_init(self, *args, **kwargs)
fd = sock.makefile() _request_instances[self] = None
fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
'Host: localhost\r\n' with mock.patch.object(Request, "__init__", request_init):
'Connection: close\r\n' prolis = _test_sockets[0]
'X-Auth-Token: t\r\n' prosrv = _test_servers[0]
'Content-Length: %s\r\n' obj_len = prosrv.client_chunk_size * 2
'Content-Type: application/octet-stream\r\n' # PUT test file
'\r\n%s' % (obj_len, 'a' * obj_len)) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
fd.flush() fd = sock.makefile()
headers = readuntil2crlfs(fd) fd.write('PUT /v1/a/c/test_leak_1 HTTP/1.1\r\n'
exp = 'HTTP/1.1 201' 'Host: localhost\r\n'
self.assertEqual(headers[:len(exp)], exp) 'Connection: close\r\n'
# Remember Request instance count, make sure the GC is run for pythons 'X-Auth-Token: t\r\n'
# without reference counting. 'Content-Length: %s\r\n'
for i in xrange(4): 'Content-Type: application/octet-stream\r\n'
sleep(0) # let eventlet do its thing '\r\n%s' % (obj_len, 'a' * obj_len))
gc.collect() fd.flush()
else: headers = readuntil2crlfs(fd)
sleep(0) exp = 'HTTP/1.1 201'
before_request_instances = len(_request_instances) self.assertEqual(headers[:len(exp)], exp)
# GET test file, but disconnect early # Remember Request instance count, make sure the GC is run for
sock = connect_tcp(('localhost', prolis.getsockname()[1])) # pythons without reference counting.
fd = sock.makefile() for i in xrange(4):
fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n' sleep(0) # let eventlet do its thing
'Host: localhost\r\n' gc.collect()
'Connection: close\r\n' else:
'X-Auth-Token: t\r\n' sleep(0)
'\r\n') before_request_instances = len(_request_instances)
fd.flush() # GET test file, but disconnect early
headers = readuntil2crlfs(fd) sock = connect_tcp(('localhost', prolis.getsockname()[1]))
exp = 'HTTP/1.1 200' fd = sock.makefile()
self.assertEqual(headers[:len(exp)], exp) fd.write('GET /v1/a/c/test_leak_1 HTTP/1.1\r\n'
fd.read(1) 'Host: localhost\r\n'
fd.close() 'Connection: close\r\n'
sock.close() 'X-Auth-Token: t\r\n'
# Make sure the GC is run again for pythons without reference counting '\r\n')
for i in xrange(4): fd.flush()
sleep(0) # let eventlet do its thing headers = readuntil2crlfs(fd)
gc.collect() exp = 'HTTP/1.1 200'
else: self.assertEqual(headers[:len(exp)], exp)
sleep(0) fd.read(1)
self.assertEquals(before_request_instances, len(_request_instances)) fd.close()
sock.close()
# Make sure the GC is run again for pythons without reference
# counting
for i in xrange(4):
sleep(0) # let eventlet do its thing
gc.collect()
else:
sleep(0)
self.assertEquals(
before_request_instances, len(_request_instances))
def test_OPTIONS(self): def test_OPTIONS(self):
with save_globals(): with save_globals():
@ -4926,9 +4526,10 @@ class TestContainerController(unittest.TestCase):
controller = \ controller = \
proxy_server.ContainerController(self.app, 'a', 'c') proxy_server.ContainerController(self.app, 'a', 'c')
set_http_connect(200, 201, 201, 201, give_connect=test_connect) set_http_connect(200, 201, 201, 201, give_connect=test_connect)
req = Request.blank('/v1/a/c', req = Request.blank(
environ={'REQUEST_METHOD': method}, '/v1/a/c',
headers={test_header: test_value}) environ={'REQUEST_METHOD': method, 'swift_owner': True},
headers={test_header: test_value})
self.app.update_request(req) self.app.update_request(req)
getattr(controller, method)(req) getattr(controller, method)(req)
self.assertEquals(test_errors, []) self.assertEquals(test_errors, [])
@ -6006,313 +5607,6 @@ class Stub(object):
pass pass
class TestSegmentedIterable(unittest.TestCase):
def setUp(self):
self.controller = FakeObjectController()
def test_load_next_segment_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception,
SegmentedIterable(self.controller, None,
[None])._load_next_segment)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_load_next_segment_with_no_segments(self):
self.assertRaises(StopIteration,
SegmentedIterable(self.controller, 'lc',
[])._load_next_segment)
def test_load_next_segment_with_one_segment(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[0][4], '/a/lc/o1')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '1')
def test_load_next_segment_with_two_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o1')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '1')
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '22')
def test_load_next_segment_rate_limiting(self):
sleep_calls = []
def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep
try:
swift.proxy.controllers.obj.sleep = _stub_sleep
segit = SegmentedIterable(
self.controller, 'lc', [
{'name': 'o1'}, {'name': 'o2'}, {'name': 'o3'},
{'name': 'o4'}, {'name': 'o5'}])
# rate_limit_after_segment == 3, so the first 3 segments should
# invoke no sleeping.
for _ in xrange(3):
segit._load_next_segment()
self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o3')
# Loading of next (4th) segment starts rate-limiting.
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o4')
sleep_calls = []
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o5')
finally:
swift.proxy.controllers.obj.sleep = orig_sleep
def test_load_next_segment_range_req_rate_limiting(self):
sleep_calls = []
def _stub_sleep(sleepy_time):
sleep_calls.append(sleepy_time)
orig_sleep = swift.proxy.controllers.obj.sleep
try:
swift.proxy.controllers.obj.sleep = _stub_sleep
segit = SegmentedIterable(
self.controller, 'lc', [
{'name': 'o0', 'bytes': 5}, {'name': 'o1', 'bytes': 5},
{'name': 'o2', 'bytes': 1}, {'name': 'o3'}, {'name': 'o4'},
{'name': 'o5'}, {'name': 'o6'}])
# this tests for a range request which skips over the whole first
# segment, after that 3 segments will be read in because the
# rate_limit_after_segment == 3, then sleeping starts
segit_iter = segit.app_iter_range(10, None)
segit_iter.next()
for _ in xrange(2):
# this is set to 2 instead of 3 because o2 was loaded after
# o0 and o1 were skipped.
segit._load_next_segment()
self.assertEquals([], sleep_calls)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o4')
# Loading of next (5th) segment starts rate-limiting.
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o5')
sleep_calls = []
segit._load_next_segment()
self.assertAlmostEqual(0.5, sleep_calls[0], places=2)
self.assertEquals(self.controller.GETorHEAD_base_args[-1][4],
'/a/lc/o6')
finally:
swift.proxy.controllers.obj.sleep = orig_sleep
def test_load_next_segment_with_two_segments_skip_first(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit.ratelimit_index = 0
segit.listing.next()
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '22')
def test_load_next_segment_with_seek(self):
segit = SegmentedIterable(self.controller, 'lc',
[{'name': 'o1', 'bytes': 1},
{'name': 'o2', 'bytes': 2}])
segit.ratelimit_index = 0
segit.listing.next()
segit.seek = 1
segit._load_next_segment()
self.assertEquals(
self.controller.GETorHEAD_base_args[-1][4], '/a/lc/o2')
self.assertEquals(
str(self.controller.GETorHEAD_base_args[-1][0].range),
'bytes=1-')
data = ''.join(segit.segment_iter)
self.assertEquals(data, '2')
def test_fetching_only_what_you_need(self):
segit = SegmentedIterable(self.controller, 'lc',
[{'name': 'o7', 'bytes': 7},
{'name': 'o8', 'bytes': 8},
{'name': 'o9', 'bytes': 9}])
body = ''.join(segit.app_iter_range(10, 20))
self.assertEqual('8888899999', body)
GoH_args = self.controller.GETorHEAD_base_args
self.assertEquals(2, len(GoH_args))
# Either one is fine, as they both indicate "from byte 3 to (the last)
# byte 8".
self.assert_(str(GoH_args[0][0].range) in ['bytes=3-', 'bytes=3-8'])
# This one must ask only for the bytes it needs; otherwise we waste
# bandwidth pulling bytes from the object server and then throwing
# them out
self.assertEquals(str(GoH_args[1][0].range), 'bytes=0-4')
def test_load_next_segment_with_get_error(self):
def local_GETorHEAD_base(*args):
return HTTPNotFound()
self.controller.GETorHEAD_base = local_GETorHEAD_base
self.assertRaises(Exception,
SegmentedIterable(self.controller, 'lc',
[{'name': 'o1'}])._load_next_segment)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404')
def test_iter_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception, ''.join,
SegmentedIterable(self.controller, None, [None]))
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_iter_with_no_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [])
self.assertEquals(''.join(segit), '')
def test_iter_with_one_segment(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}])
segit.response = Stub()
self.assertEquals(''.join(segit), '1')
def test_iter_with_two_segments(self):
segit = SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}, {'name': 'o2'}])
segit.response = Stub()
self.assertEquals(''.join(segit), '122')
def test_iter_with_get_error(self):
def local_GETorHEAD_base(*args):
return HTTPNotFound()
self.controller.GETorHEAD_base = local_GETorHEAD_base
self.assertRaises(Exception, ''.join,
SegmentedIterable(self.controller, 'lc', [{'name':
'o1'}]))
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
self.assertEquals(str(self.controller.exception_info[1]),
'Could not load object segment /a/lc/o1: 404')
def test_app_iter_range_unexpected_error(self):
# Iterator value isn't a dict
self.assertRaises(Exception,
SegmentedIterable(self.controller, None,
[None]).app_iter_range(None,
None).next)
self.assert_(self.controller.exception_args[0].startswith(
'ERROR: While processing manifest'))
def test_app_iter_range_with_no_segments(self):
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(None, None)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(3, None)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(3, 5)), '')
self.assertEquals(''.join(SegmentedIterable(
self.controller, 'lc', []).app_iter_range(None, 5)), '')
def test_app_iter_range_with_one_segment(self):
listing = [{'name': 'o1', 'bytes': 1}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)), '1')
segit = SegmentedIterable(self.controller, 'lc', listing)
self.assertEquals(''.join(segit.app_iter_range(3, None)), '')
segit = SegmentedIterable(self.controller, 'lc', listing)
self.assertEquals(''.join(segit.app_iter_range(3, 5)), '')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 5)), '1')
def test_app_iter_range_with_two_segments(self):
listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)), '122')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(1, None)), '22')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(1, 5)), '22')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 2)), '12')
def test_app_iter_range_with_many_segments(self):
listing = [{'name': 'o1', 'bytes': 1}, {'name': 'o2', 'bytes': 2},
{'name': 'o3', 'bytes': 3}, {'name': 'o4', 'bytes': 4},
{'name': 'o5', 'bytes': 5}]
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, None)),
'122333444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(3, None)),
'333444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(5, None)), '3444455555')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 6)), '122333')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(None, 7)), '1223334')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(3, 7)), '3334')
segit = SegmentedIterable(self.controller, 'lc', listing)
segit.response = Stub()
self.assertEquals(''.join(segit.app_iter_range(5, 7)), '34')
class TestProxyObjectPerformance(unittest.TestCase): class TestProxyObjectPerformance(unittest.TestCase):
def setUp(self): def setUp(self):

1
tools/requirements.txt
View File

@ -5,4 +5,3 @@ netifaces>=0.5
pastedeploy>=1.3.3 pastedeploy>=1.3.3
simplejson>=2.0.9 simplejson>=2.0.9
xattr>=0.4 xattr>=0.4
python-swiftclient

6
tools/test-requires
View File

@ -1,7 +1,4 @@
# Install bounded pep8/pyflakes first, then let flake8 install # Hacking already pins down pep8, pyflakes and flake8
pep8==1.4.5
pyflakes==0.7.2
flake8==2.0
hacking>=0.5.6,<0.6 hacking>=0.5.6,<0.6
coverage coverage
nose nose
@ -10,5 +7,6 @@ openstack.nose_plugin
nosehtmloutput nosehtmloutput
sphinx>=1.1.2,<1.2 sphinx>=1.1.2,<1.2
mock>=0.8.0 mock>=0.8.0
python-swiftclient
python-keystoneclient python-keystoneclient
prettytable prettytable

2
tox.ini
View File

@ -15,7 +15,7 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_OPENSTACK_SHOW_ELAPSED=1 NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1 NOSE_OPENSTACK_STDOUT=1
deps = deps =
https://launchpad.net/gluster-swift/icehouse/1.12.0/+download/swift-1.12.0.tar.gz https://launchpad.net/gluster-swift/icehouse/1.13.0/+download/swift-1.13.0.tar.gz
--download-cache={homedir}/.pipcache --download-cache={homedir}/.pipcache
-r{toxinidir}/tools/test-requires -r{toxinidir}/tools/test-requires
-r{toxinidir}/tools/requirements.txt -r{toxinidir}/tools/requirements.txt