x/swiftonhpss
Code Issues Proposed changes

Merge upstream updates to support the new Liberty release

Browse Source 
Conflicts:
	swiftonhpss.spec
	swiftonhpss/swift/__init__.py
	test/functional/tests.py
This commit is contained in:
Phil Bridges 2016-01-05 10:27:31 -06:00
commit 14599e460b
15 changed files with 1458 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements.txt
View File

@ -8,6 +8,7 @@ greenlet>=0.3.1
netifaces>=0.5,!=0.10.0,!=0.10.1 netifaces>=0.5,!=0.10.0,!=0.10.1
pastedeploy>=1.3.3 pastedeploy>=1.3.3
simplejson>=2.0.9 simplejson>=2.0.9
six>=1.9.0
xattr>=0.4 xattr>=0.4
PyECLib==1.0.7 PyECLib==1.0.7

5
swiftonhpss.spec
View File

@ -17,7 +17,7 @@ BuildRequires: python-devel
BuildRequires: python-setuptools BuildRequires: python-setuptools
Requires : python Requires : python
Requires : python-setuptools Requires : python-setuptools
Requires : openstack-swift-object = 2.3.0 Requires : openstack-swift-object = 2.5.0
%description %description
SwiftOnHPSS is a Swift Object Server implementation that enables users to SwiftOnHPSS is a Swift Object Server implementation that enables users to
@ -58,6 +58,9 @@ cp -r etc/* %{buildroot}/%{_confdir}/
rm -rf %{buildroot} rm -rf %{buildroot}
%changelog %changelog
* Wed Dec 23 2015 Prashanth Pai <ppai@redhat.com> - 2.5.0-0
- Update spec file to support Liberty release of Swift
* Thu Dec 10 2015 Phil Bridges <pgbridge@us.ibm.com> * Thu Dec 10 2015 Phil Bridges <pgbridge@us.ibm.com>
- Fork SwiftOnFile into SwiftOnHPSS, add HPSS-specific features - Fork SwiftOnFile into SwiftOnHPSS, add HPSS-specific features

2
swiftonhpss/swift/__init__.py
View File

@ -43,7 +43,7 @@ class PkgInfo(object):
# Change the Package version here # Change the Package version here
_pkginfo = PkgInfo(canonical_version='2.3.0', _pkginfo = PkgInfo(canonical_version='2.5.0',
release='0', release='0',
name='swiftonhpss', name='swiftonhpss',
final=False) final=False)

19
swiftonhpss/swift/obj/diskfile.py
View File

@ -30,7 +30,7 @@ from contextlib import contextmanager
from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \ from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \
AlreadyExistsAsDir AlreadyExistsAsDir
from swift.common.utils import ThreadPool, hash_path, \ from swift.common.utils import ThreadPool, hash_path, \
normalize_timestamp, fallocate normalize_timestamp, fallocate, Timestamp
from swift.common.exceptions import DiskFileNotExist, DiskFileError, \ from swift.common.exceptions import DiskFileNotExist, DiskFileError, \
DiskFileNoSpace, DiskFileDeviceUnavailable, DiskFileNotOpen, \ DiskFileNoSpace, DiskFileDeviceUnavailable, DiskFileNotOpen, \
DiskFileExpired DiskFileExpired
@ -53,7 +53,7 @@ from swift.obj.diskfile import get_async_dir
# FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will # FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will
# be back ported. See http://www.python.org/dev/peps/pep-0433/ # be back ported. See http://www.python.org/dev/peps/pep-0433/
O_CLOEXEC = 02000000 O_CLOEXEC = 0o2000000
MAX_RENAME_ATTEMPTS = 10 MAX_RENAME_ATTEMPTS = 10
MAX_OPEN_ATTEMPTS = 10 MAX_OPEN_ATTEMPTS = 10
@ -583,7 +583,7 @@ class DiskFile(object):
""" """
def __init__(self, mgr, dev_path, threadpool, partition, def __init__(self, mgr, dev_path, threadpool, partition,
account=None, container=None, obj=None, account=None, container=None, obj=None,
policy=None, uid=DEFAULT_UID, gid=DEFAULT_GID): policy=None, uid=DEFAULT_UID, gid=DEFAULT_GID, **kwargs):
# Variables partition and policy is currently unused. # Variables partition and policy is currently unused.
self._mgr = mgr self._mgr = mgr
self._device_path = dev_path self._device_path = dev_path
@ -617,6 +617,18 @@ class DiskFile(object):
self._data_file = os.path.join(self._put_datadir, self._obj) self._data_file = os.path.join(self._put_datadir, self._obj)
@property
def timestamp(self):
if self._metadata is None:
raise DiskFileNotOpen()
return Timestamp(self._metadata.get('X-Timestamp'))
@property
def data_timestamp(self):
if self._metadata is None:
raise DiskFileNotOpen()
return Timestamp(self._metadata.get('X-Timestamp'))
def open(self): def open(self):
""" """
Open the object. Open the object.
@ -733,7 +745,6 @@ class DiskFile(object):
the REST API *before* the object has actually been read. It is the the REST API *before* the object has actually been read. It is the
responsibility of the implementation to properly handle that. responsibility of the implementation to properly handle that.
""" """
self._metadata = None
self._close_fd() self._close_fd()
def get_metadata(self): def get_metadata(self):

2
test-requirements.txt
View File

@ -3,7 +3,7 @@
# process, which may cause wedges in the gate later. # process, which may cause wedges in the gate later.
# Hacking already pins down pep8, pyflakes and flake8 # Hacking already pins down pep8, pyflakes and flake8
hacking>=0.8.0,<0.9 hacking>=0.10.0,<0.11
coverage coverage
nose nose
nosexcover nosexcover

17
test/__init__.py
View File

@ -15,7 +15,7 @@
# See http://code.google.com/p/python-nose/issues/detail?id=373 # See http://code.google.com/p/python-nose/issues/detail?id=373
# The code below enables nosetests to work with i18n _() blocks # The code below enables nosetests to work with i18n _() blocks
from __future__ import print_function
import sys import sys
import os import os
try: try:
@ -63,15 +63,12 @@ def get_config(section_name=None, defaults=None):
config = readconf(config_file, section_name) config = readconf(config_file, section_name)
except SystemExit: except SystemExit:
if not os.path.exists(config_file): if not os.path.exists(config_file):
print >>sys.stderr, \ print('Unable to read test config %s - file not found'
'Unable to read test config %s - file not found' \ % config_file, file=sys.stderr)
% config_file
elif not os.access(config_file, os.R_OK): elif not os.access(config_file, os.R_OK):
print >>sys.stderr, \ print('Unable to read test config %s - permission denied'
'Unable to read test config %s - permission denied' \ % config_file, file=sys.stderr)
% config_file
else: else:
print >>sys.stderr, \ print('Unable to read test config %s - section %s not found'
'Unable to read test config %s - section %s not found' \ % (config_file, section_name), file=sys.stderr)
% (config_file, section_name)
return config return config

96
test/functional/__init__.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import print_function
import mock import mock
import os import os
import sys import sys
@ -23,20 +24,24 @@ import eventlet
import eventlet.debug import eventlet.debug
import functools import functools
import random import random
from ConfigParser import ConfigParser, NoSectionError
from time import time, sleep from time import time, sleep
from httplib import HTTPException
from urlparse import urlparse from urlparse import urlparse
from nose import SkipTest from nose import SkipTest
from contextlib import closing from contextlib import closing
from gzip import GzipFile from gzip import GzipFile
from shutil import rmtree from shutil import rmtree
from tempfile import mkdtemp from tempfile import mkdtemp
from six.moves.configparser import ConfigParser, NoSectionError
from six.moves import http_client
from six.moves.http_client import HTTPException
from swift.common.middleware.memcache import MemcacheMiddleware from swift.common.middleware.memcache import MemcacheMiddleware
from swift.common.storage_policy import parse_storage_policies, PolicyError from swift.common.storage_policy import parse_storage_policies, PolicyError
from test import get_config from test import get_config
from test.functional.swift_test_client import Account, Connection, \ from test.functional.swift_test_client import Account, Connection, Container, \
ResponseError ResponseError
# This has the side effect of mocking out the xattr module so that unit tests # This has the side effect of mocking out the xattr module so that unit tests
# (and in this case, when in-process functional tests are called for) can run # (and in this case, when in-process functional tests are called for) can run
@ -46,13 +51,13 @@ from test.unit import debug_logger, FakeMemcache
from swift.common import constraints, utils, ring, storage_policy from swift.common import constraints, utils, ring, storage_policy
from swift.common.ring import Ring from swift.common.ring import Ring
from swift.common.wsgi import monkey_patch_mimetools, loadapp from swift.common.wsgi import monkey_patch_mimetools, loadapp
from swift.common.utils import config_true_value from swift.common.utils import config_true_value, split_path
from swift.account import server as account_server from swift.account import server as account_server
from swift.container import server as container_server from swift.container import server as container_server
from swift.obj import server as object_server, mem_server as mem_object_server from swift.obj import server as object_server, mem_server as mem_object_server
import swift.proxy.controllers.obj import swift.proxy.controllers.obj
http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT
DEBUG = True DEBUG = True
# In order to get the proper blocking behavior of sockets without using # In order to get the proper blocking behavior of sockets without using
@ -105,6 +110,7 @@ orig_swift_conf_name = None
in_process = False in_process = False
_testdir = _test_servers = _test_coros = None _testdir = _test_servers = _test_coros = None
policy_specified = None
class FakeMemcacheMiddleware(MemcacheMiddleware): class FakeMemcacheMiddleware(MemcacheMiddleware):
@ -123,7 +129,7 @@ class InProcessException(BaseException):
def _info(msg): def _info(msg):
print >> sys.stderr, msg print(msg, file=sys.stderr)
def _debug(msg): def _debug(msg):
@ -209,7 +215,6 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
for policy in policies: for policy in policies:
conf.remove_section(sp_prefix + str(policy.idx)) conf.remove_section(sp_prefix + str(policy.idx))
policy_specified = os.environ.get('SWIFT_TEST_POLICY')
if policy_specified: if policy_specified:
policy_to_test = policies.get_by_name(policy_specified) policy_to_test = policies.get_by_name(policy_specified)
if policy_to_test is None: if policy_to_test is None:
@ -497,7 +502,7 @@ def get_cluster_info():
# Most likely the swift cluster has "expose_info = false" set # Most likely the swift cluster has "expose_info = false" set
# in its proxy-server.conf file, so we'll just do the best we # in its proxy-server.conf file, so we'll just do the best we
# can. # can.
print >>sys.stderr, "** Swift Cluster not exposing /info **" print("** Swift Cluster not exposing /info **", file=sys.stderr)
# Finally, we'll allow any constraint present in the swift-constraints # Finally, we'll allow any constraint present in the swift-constraints
# section of test.conf to override everything. Note that only those # section of test.conf to override everything. Note that only those
@ -509,8 +514,8 @@ def get_cluster_info():
except KeyError: except KeyError:
pass pass
except ValueError: except ValueError:
print >>sys.stderr, "Invalid constraint value: %s = %s" % ( print("Invalid constraint value: %s = %s" % (
k, test_constraints[k]) k, test_constraints[k]), file=sys.stderr)
eff_constraints.update(test_constraints) eff_constraints.update(test_constraints)
# Just make it look like these constraints were loaded from a /info call, # Just make it look like these constraints were loaded from a /info call,
@ -520,6 +525,9 @@ def get_cluster_info():
def setup_package(): def setup_package():
global policy_specified
policy_specified = os.environ.get('SWIFT_TEST_POLICY')
in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS') in_process_env = os.environ.get('SWIFT_TEST_IN_PROCESS')
if in_process_env is not None: if in_process_env is not None:
use_in_process = utils.config_true_value(in_process_env) use_in_process = utils.config_true_value(in_process_env)
@ -557,8 +565,8 @@ def setup_package():
in_process_setup(the_object_server=( in_process_setup(the_object_server=(
mem_object_server if in_mem_obj else object_server)) mem_object_server if in_mem_obj else object_server))
except InProcessException as exc: except InProcessException as exc:
print >> sys.stderr, ('Exception during in-process setup: %s' print(('Exception during in-process setup: %s'
% str(exc)) % str(exc)), file=sys.stderr)
raise raise
global web_front_end global web_front_end
@ -667,20 +675,19 @@ def setup_package():
global skip global skip
skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
if skip: if skip:
print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' print('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG', file=sys.stderr)
global skip2 global skip2
skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
if not skip and skip2: if not skip and skip2:
print >>sys.stderr, \ print('SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS '
'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS' \ 'DUE TO NO CONFIG FOR THEM', file=sys.stderr)
' DUE TO NO CONFIG FOR THEM'
global skip3 global skip3
skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
if not skip and skip3: if not skip and skip3:
print >>sys.stderr, \ print('SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS'
'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' 'DUE TO NO CONFIG FOR THEM', file=sys.stderr)
global skip_if_not_v3 global skip_if_not_v3
skip_if_not_v3 = (swift_test_auth_version != '3' skip_if_not_v3 = (swift_test_auth_version != '3'
@ -688,16 +695,33 @@ def setup_package():
swift_test_user[3], swift_test_user[3],
swift_test_key[3]])) swift_test_key[3]]))
if not skip and skip_if_not_v3: if not skip and skip_if_not_v3:
print >>sys.stderr, \ print('SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3',
'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3' file=sys.stderr)
global skip_service_tokens global skip_service_tokens
skip_service_tokens = not all([not skip, swift_test_user[4], skip_service_tokens = not all([not skip, swift_test_user[4],
swift_test_key[4], swift_test_tenant[4], swift_test_key[4], swift_test_tenant[4],
swift_test_service_prefix]) swift_test_service_prefix])
if not skip and skip_service_tokens: if not skip and skip_service_tokens:
print >>sys.stderr, \ print(
'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS' 'SKIPPING FUNCTIONAL TESTS SPECIFIC TO SERVICE TOKENS',
file=sys.stderr)
if policy_specified:
policies = FunctionalStoragePolicyCollection.from_info()
for p in policies:
# policy names are case-insensitive
if policy_specified.lower() == p['name'].lower():
_info('Using specified policy %s' % policy_specified)
FunctionalStoragePolicyCollection.policy_specified = p
Container.policy_specified = policy_specified
break
else:
_info(
'SKIPPING FUNCTIONAL TESTS: Failed to find specified policy %s'
% policy_specified)
raise Exception('Failed to find specified policy %s'
% policy_specified)
get_cluster_info() get_cluster_info()
@ -746,8 +770,24 @@ conn = [None, None, None, None, None]
def connection(url): def connection(url):
if has_insecure: if has_insecure:
return http_connection(url, insecure=insecure) parsed_url, http_conn = http_connection(url, insecure=insecure)
return http_connection(url) else:
parsed_url, http_conn = http_connection(url)
orig_request = http_conn.request
# Add the policy header if policy_specified is set
def request_with_policy(method, url, body=None, headers={}):
version, account, container, obj = split_path(url, 1, 4, True)
if policy_specified and method == 'PUT' and container and not obj \
and 'X-Storage-Policy' not in headers:
headers['X-Storage-Policy'] = policy_specified
return orig_request(method, url, body, headers)
http_conn.request = request_with_policy
return parsed_url, http_conn
def get_url_token(user_index, os_options): def get_url_token(user_index, os_options):
@ -898,6 +938,9 @@ def requires_acls(f):
class FunctionalStoragePolicyCollection(object): class FunctionalStoragePolicyCollection(object):
# policy_specified is set in __init__.py when tests are being set up.
policy_specified = None
def __init__(self, policies): def __init__(self, policies):
self._all = policies self._all = policies
self.default = None self.default = None
@ -939,6 +982,11 @@ class FunctionalStoragePolicyCollection(object):
p.get(k) != v for k, v in kwargs.items())]) p.get(k) != v for k, v in kwargs.items())])
def select(self): def select(self):
# check that a policy was specified and that it is available
# in the current list (i.e., hasn't been excluded of the current list)
if self.policy_specified and self.policy_specified in self:
return self.policy_specified
else:
return random.choice(self) return random.choice(self)

87
test/functional/swift_test_client.py
View File

@ -14,25 +14,27 @@
# limitations under the License. # limitations under the License.
import hashlib import hashlib
import httplib
import os import os
import random import random
import socket import socket
import StringIO
import time import time
import urllib import urllib
import simplejson as json import simplejson as json
from nose import SkipTest from nose import SkipTest
from xml.dom import minidom from xml.dom import minidom
import six
from six.moves import http_client
from swiftclient import get_auth from swiftclient import get_auth
from swift.common import constraints
from swift.common.utils import config_true_value from swift.common.utils import config_true_value
from test import safe_repr from test import safe_repr
http_client._MAXHEADERS = constraints.MAX_HEADER_COUNT
class AuthenticationFailed(Exception): class AuthenticationFailed(Exception):
pass pass
@ -68,7 +70,7 @@ class ResponseError(Exception):
def listing_empty(method): def listing_empty(method):
for i in xrange(6): for i in range(6):
if len(method()) == 0: if len(method()) == 0:
return True return True
@ -163,10 +165,10 @@ class Connection(object):
x = storage_url.split('/') x = storage_url.split('/')
if x[0] == 'http:': if x[0] == 'http:':
self.conn_class = httplib.HTTPConnection self.conn_class = http_client.HTTPConnection
self.storage_port = 80 self.storage_port = 80
elif x[0] == 'https:': elif x[0] == 'https:':
self.conn_class = httplib.HTTPSConnection self.conn_class = http_client.HTTPSConnection
self.storage_port = 443 self.storage_port = 443
else: else:
raise ValueError('unexpected protocol %s' % (x[0])) raise ValueError('unexpected protocol %s' % (x[0]))
@ -181,7 +183,11 @@ class Connection(object):
self.storage_url = str('/%s/%s' % (x[3], x[4])) self.storage_url = str('/%s/%s' % (x[3], x[4]))
self.account_name = str(x[4]) self.account_name = str(x[4])
self.auth_user = auth_user self.auth_user = auth_user
self.storage_token = storage_token # With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username) self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect() self.http_connect()
@ -230,6 +236,9 @@ class Connection(object):
if not cfg.get('no_auth_token'): if not cfg.get('no_auth_token'):
headers['X-Auth-Token'] = self.storage_token headers['X-Auth-Token'] = self.storage_token
if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token')
if isinstance(hdrs, dict): if isinstance(hdrs, dict):
headers.update(hdrs) headers.update(hdrs)
return headers return headers
@ -276,7 +285,7 @@ class Connection(object):
try: try:
self.response = try_request() self.response = try_request()
except httplib.HTTPException as e: except http_client.HTTPException as e:
fail_messages.append(safe_repr(e)) fail_messages.append(safe_repr(e))
continue continue
@ -330,7 +339,7 @@ class Connection(object):
port=self.storage_port) port=self.storage_port)
# self.connection.set_debuglevel(3) # self.connection.set_debuglevel(3)
self.connection.putrequest('PUT', path) self.connection.putrequest('PUT', path)
for key, value in headers.iteritems(): for key, value in headers.items():
self.connection.putheader(key, value) self.connection.putheader(key, value)
self.connection.endheaders() self.connection.endheaders()
@ -481,6 +490,9 @@ class Account(Base):
class Container(Base): class Container(Base):
# policy_specified is set in __init__.py when tests are being set up.
policy_specified = None
def __init__(self, conn, account, name): def __init__(self, conn, account, name):
self.conn = conn self.conn = conn
self.account = str(account) self.account = str(account)
@ -493,9 +505,23 @@ class Container(Base):
parms = {} parms = {}
if cfg is None: if cfg is None:
cfg = {} cfg = {}
if self.policy_specified and 'X-Storage-Policy' not in hdrs:
hdrs['X-Storage-Policy'] = self.policy_specified
return self.conn.make_request('PUT', self.path, hdrs=hdrs, return self.conn.make_request('PUT', self.path, hdrs=hdrs,
parms=parms, cfg=cfg) in (201, 202) parms=parms, cfg=cfg) in (201, 202)
def update_metadata(self, hdrs=None, cfg=None):
if hdrs is None:
hdrs = {}
if cfg is None:
cfg = {}
self.conn.make_request('POST', self.path, hdrs=hdrs, cfg=cfg)
if not 200 <= self.conn.response.status <= 299:
raise ResponseError(self.conn.response, 'POST',
self.conn.make_path(self.path))
return True
def delete(self, hdrs=None, parms=None): def delete(self, hdrs=None, parms=None):
if hdrs is None: if hdrs is None:
hdrs = {} hdrs = {}
@ -626,6 +652,9 @@ class File(Base):
else: else:
headers['Content-Length'] = 0 headers['Content-Length'] = 0
if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token')
if cfg.get('no_content_type'): if cfg.get('no_content_type'):
pass pass
elif self.content_type: elif self.content_type:
@ -643,7 +672,7 @@ class File(Base):
block_size = 4096 block_size = 4096
if isinstance(data, str): if isinstance(data, str):
data = StringIO.StringIO(data) data = six.StringIO(data)
checksum = hashlib.md5() checksum = hashlib.md5()
buff = data.read(block_size) buff = data.read(block_size)
@ -700,13 +729,13 @@ class File(Base):
return self.conn.make_request('COPY', self.path, hdrs=headers, return self.conn.make_request('COPY', self.path, hdrs=headers,
parms=parms) == 201 parms=parms) == 201
def delete(self, hdrs=None, parms=None): def delete(self, hdrs=None, parms=None, cfg=None):
if hdrs is None: if hdrs is None:
hdrs = {} hdrs = {}
if parms is None: if parms is None:
parms = {} parms = {}
if self.conn.make_request('DELETE', self.path, hdrs=hdrs, if self.conn.make_request('DELETE', self.path, hdrs=hdrs,
parms=parms) != 204: cfg=cfg, parms=parms) != 204:
raise ResponseError(self.conn.response, 'DELETE', raise ResponseError(self.conn.response, 'DELETE',
self.conn.make_path(self.path)) self.conn.make_path(self.path))
@ -847,7 +876,7 @@ class File(Base):
finally: finally:
fobj.close() fobj.close()
def sync_metadata(self, metadata=None, cfg=None): def sync_metadata(self, metadata=None, cfg=None, parms=None):
if metadata is None: if metadata is None:
metadata = {} metadata = {}
if cfg is None: if cfg is None:
@ -864,7 +893,8 @@ class File(Base):
else: else:
headers['Content-Length'] = 0 headers['Content-Length'] = 0
self.conn.make_request('POST', self.path, hdrs=headers, cfg=cfg) self.conn.make_request('POST', self.path, hdrs=headers,
parms=parms, cfg=cfg)
if self.conn.response.status not in (201, 202): if self.conn.response.status not in (201, 202):
raise ResponseError(self.conn.response, 'POST', raise ResponseError(self.conn.response, 'POST',
@ -917,7 +947,7 @@ class File(Base):
pass pass
self.size = int(os.fstat(data.fileno())[6]) self.size = int(os.fstat(data.fileno())[6])
else: else:
data = StringIO.StringIO(data) data = six.StringIO(data)
self.size = data.len self.size = data.len
headers = self.make_headers(cfg=cfg) headers = self.make_headers(cfg=cfg)
@ -969,7 +999,7 @@ class File(Base):
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
raise ResponseError(self.conn.response, 'PUT', raise ResponseError(self.conn.response, 'PUT',
self.conn.make_path(self.path)) self.conn.make_path(self.path))
self.md5 = self.compute_md5sum(StringIO.StringIO(data)) self.md5 = self.compute_md5sum(six.StringIO(data))
return data return data
def write_random_return_resp(self, size=None, hdrs=None, parms=None, def write_random_return_resp(self, size=None, hdrs=None, parms=None,
@ -986,5 +1016,28 @@ class File(Base):
return_resp=True) return_resp=True)
if not resp: if not resp:
raise ResponseError(self.conn.response) raise ResponseError(self.conn.response)
self.md5 = self.compute_md5sum(StringIO.StringIO(data)) self.md5 = self.compute_md5sum(six.StringIO(data))
return resp return resp
def post(self, hdrs=None, parms=None, cfg=None, return_resp=False):
if hdrs is None:
hdrs = {}
if parms is None:
parms = {}
if cfg is None:
cfg = {}
headers = self.make_headers(cfg=cfg)
headers.update(hdrs)
self.conn.make_request('POST', self.path, hdrs=headers,
parms=parms, cfg=cfg)
if self.conn.response.status not in (201, 202):
raise ResponseError(self.conn.response, 'POST',
self.conn.make_path(self.path))
if return_resp:
return self.conn.response
return True

79
test/functional/test_account.py
View File

@ -21,6 +21,7 @@ from uuid import uuid4
from nose import SkipTest from nose import SkipTest
from string import letters from string import letters
from six.moves import range
from swift.common.middleware.acl import format_acl from swift.common.middleware.acl import format_acl
from test.functional import check_response, retry, requires_acls, \ from test.functional import check_response, retry, requires_acls, \
@ -88,22 +89,22 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-test'), None) self.assertEqual(resp.getheader('x-account-meta-test'), None)
resp = retry(get) resp = retry(get)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-test'), None) self.assertEqual(resp.getheader('x-account-meta-test'), None)
resp = retry(post, 'Value') resp = retry(post, 'Value')
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') self.assertEqual(resp.getheader('x-account-meta-test'), 'Value')
resp = retry(get) resp = retry(get)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-test'), 'Value') self.assertEqual(resp.getheader('x-account-meta-test'), 'Value')
def test_invalid_acls(self): def test_invalid_acls(self):
@ -189,7 +190,7 @@ class TestAccount(unittest.TestCase):
# cannot read account # cannot read account
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read access # grant read access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -203,7 +204,7 @@ class TestAccount(unittest.TestCase):
# read-only can read account headers # read-only can read account headers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204)) self.assertIn(resp.status, (200, 204))
# but not acls # but not acls
self.assertEqual(resp.getheader('X-Account-Access-Control'), None) self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
@ -220,7 +221,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204)) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value') self.assertEqual(resp.getheader('X-Account-Meta-Test'), 'value')
@requires_acls @requires_acls
@ -240,7 +241,7 @@ class TestAccount(unittest.TestCase):
# cannot read account # cannot read account
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-write access # grant read-write access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -254,7 +255,7 @@ class TestAccount(unittest.TestCase):
# read-write can read account headers # read-write can read account headers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204)) self.assertIn(resp.status, (200, 204))
# but not acls # but not acls
self.assertEqual(resp.getheader('X-Account-Access-Control'), None) self.assertEqual(resp.getheader('X-Account-Access-Control'), None)
@ -281,7 +282,7 @@ class TestAccount(unittest.TestCase):
# cannot read account # cannot read account
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant admin access # grant admin access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -295,7 +296,7 @@ class TestAccount(unittest.TestCase):
# admin can read account headers # admin can read account headers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204)) self.assertIn(resp.status, (200, 204))
# including acls # including acls
self.assertEqual(resp.getheader('X-Account-Access-Control'), self.assertEqual(resp.getheader('X-Account-Access-Control'),
acl_json_str) acl_json_str)
@ -308,7 +309,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204)) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value)
# admin can even revoke their own access # admin can even revoke their own access
@ -320,7 +321,7 @@ class TestAccount(unittest.TestCase):
# and again, cannot read account # and again, cannot read account
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
@requires_acls @requires_acls
def test_protected_tempurl(self): def test_protected_tempurl(self):
@ -358,7 +359,8 @@ class TestAccount(unittest.TestCase):
# read-only tester3 can read account metadata # read-only tester3 can read account metadata
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), self.assertTrue(
resp.status in (200, 204),
'Expected status in (200, 204), got %s' % resp.status) 'Expected status in (200, 204), got %s' % resp.status)
self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value)
# but not temp-url-key # but not temp-url-key
@ -376,7 +378,8 @@ class TestAccount(unittest.TestCase):
# read-write tester3 can read account metadata # read-write tester3 can read account metadata
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), self.assertTrue(
resp.status in (200, 204),
'Expected status in (200, 204), got %s' % resp.status) 'Expected status in (200, 204), got %s' % resp.status)
self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value)
# but not temp-url-key # but not temp-url-key
@ -394,7 +397,8 @@ class TestAccount(unittest.TestCase):
# admin tester3 can read account metadata # admin tester3 can read account metadata
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), self.assertTrue(
resp.status in (200, 204),
'Expected status in (200, 204), got %s' % resp.status) 'Expected status in (200, 204), got %s' % resp.status)
self.assertEqual(resp.getheader('X-Account-Meta-Test'), value) self.assertEqual(resp.getheader('X-Account-Meta-Test'), value)
# including temp-url-key # including temp-url-key
@ -411,7 +415,8 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), self.assertTrue(
resp.status in (200, 204),
'Expected status in (200, 204), got %s' % resp.status) 'Expected status in (200, 204), got %s' % resp.status)
self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'), self.assertEqual(resp.getheader('X-Account-Meta-Temp-Url-Key'),
secret) secret)
@ -688,17 +693,17 @@ class TestAccount(unittest.TestCase):
if (tf.web_front_end == 'integral'): if (tf.web_front_end == 'integral'):
resp = retry(post, uni_key, '1') resp = retry(post, uni_key, '1')
resp.read() resp.read()
self.assertTrue(resp.status in (201, 204)) self.assertIn(resp.status, (201, 204))
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1')
resp = retry(post, 'X-Account-Meta-uni', uni_value) resp = retry(post, 'X-Account-Meta-uni', uni_value)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Account-Meta-uni'), self.assertEqual(resp.getheader('X-Account-Meta-uni'),
uni_value.encode('utf-8')) uni_value.encode('utf-8'))
if (tf.web_front_end == 'integral'): if (tf.web_front_end == 'integral'):
@ -707,7 +712,7 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader(uni_key.encode('utf-8')), self.assertEqual(resp.getheader(uni_key.encode('utf-8')),
uni_value.encode('utf-8')) uni_value.encode('utf-8'))
@ -729,24 +734,23 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-one'), '1') self.assertEqual(resp.getheader('x-account-meta-one'), '1')
resp = retry(post, 'X-Account-Meta-Two', '2') resp = retry(post, 'X-Account-Meta-Two', '2')
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-account-meta-one'), '1') self.assertEqual(resp.getheader('x-account-meta-one'), '1')
self.assertEqual(resp.getheader('x-account-meta-two'), '2') self.assertEqual(resp.getheader('x-account-meta-two'), '2')
def test_bad_metadata(self): def test_bad_metadata(self):
raise SkipTest('SOF constraints middleware enforces constraints.')
if tf.skip: if tf.skip:
raise SkipTest raise SkipTest
raise SkipTest('SOF constraints middleware enforces constraints.')
def post(url, token, parsed, conn, extra_headers): def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token} headers = {'X-Auth-Token': token}
headers.update(extra_headers) headers.update(extra_headers)
@ -793,13 +797,13 @@ class TestAccount(unittest.TestCase):
resp = retry(post, headers) resp = retry(post, headers)
headers = {} headers = {}
for x in xrange(self.max_meta_count): for x in range(self.max_meta_count):
headers['X-Account-Meta-%d' % x] = 'v' headers['X-Account-Meta-%d' % x] = 'v'
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
headers = {} headers = {}
for x in xrange(self.max_meta_count + 1): for x in range(self.max_meta_count + 1):
headers['X-Account-Meta-%d' % x] = 'v' headers['X-Account-Meta-%d' % x] = 'v'
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
@ -830,8 +834,23 @@ class TestAccount(unittest.TestCase):
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
# this POST includes metadata size that is over limit
headers['X-Account-Meta-k'] = \ headers['X-Account-Meta-k'] = \
'v' * (self.max_meta_overall_size - size) 'x' * (self.max_meta_overall_size - size)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
# this POST would be ok and the aggregate backend metadata
# size is on the border
headers = {'X-Account-Meta-k':
'y' * (self.max_meta_overall_size - size - 1)}
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
# this last POST would be ok by itself but takes the aggregate
# backend metadata size over limit
headers = {'X-Account-Meta-k':
'z' * (self.max_meta_overall_size - size)}
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 400) self.assertEqual(resp.status, 400)
@ -862,7 +881,7 @@ class TestAccountInNonDefaultDomain(unittest.TestCase):
resp = retry(head, use_account=4) resp = retry(head, use_account=4)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertTrue('X-Account-Project-Domain-Id' in resp.headers) self.assertIn('X-Account-Project-Domain-Id', resp.headers)
if __name__ == '__main__': if __name__ == '__main__':

205
test/functional/test_container.py
View File

@ -24,6 +24,8 @@ from test.functional import check_response, retry, requires_acls, \
load_constraint, requires_policies load_constraint, requires_policies
import test.functional as tf import test.functional as tf
from six.moves import range
class TestContainer(unittest.TestCase): class TestContainer(unittest.TestCase):
@ -70,7 +72,7 @@ class TestContainer(unittest.TestCase):
body = resp.read() body = resp.read()
if resp.status == 404: if resp.status == 404:
break break
self.assert_(resp.status // 100 == 2, resp.status) self.assertTrue(resp.status // 100 == 2, resp.status)
objs = json.loads(body) objs = json.loads(body)
if not objs: if not objs:
break break
@ -91,7 +93,7 @@ class TestContainer(unittest.TestCase):
# container may have not been created # container may have not been created
resp = retry(delete, self.container) resp = retry(delete, self.container)
resp.read() resp.read()
self.assert_(resp.status in (204, 404)) self.assertIn(resp.status, (204, 404))
def test_multi_metadata(self): def test_multi_metadata(self):
if tf.skip: if tf.skip:
@ -112,14 +114,14 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-one'), '1') self.assertEqual(resp.getheader('x-container-meta-one'), '1')
resp = retry(post, 'X-Container-Meta-Two', '2') resp = retry(post, 'X-Container-Meta-Two', '2')
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-one'), '1') self.assertEqual(resp.getheader('x-container-meta-one'), '1')
self.assertEqual(resp.getheader('x-container-meta-two'), '2') self.assertEqual(resp.getheader('x-container-meta-two'), '2')
@ -145,14 +147,14 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1') self.assertEqual(resp.getheader(uni_key.encode('utf-8')), '1')
resp = retry(post, 'X-Container-Meta-uni', uni_value) resp = retry(post, 'X-Container-Meta-uni', uni_value)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('X-Container-Meta-uni'), self.assertEqual(resp.getheader('X-Container-Meta-uni'),
uni_value.encode('utf-8')) uni_value.encode('utf-8'))
if (tf.web_front_end == 'integral'): if (tf.web_front_end == 'integral'):
@ -161,7 +163,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader(uni_key.encode('utf-8')), self.assertEqual(resp.getheader(uni_key.encode('utf-8')),
uni_value.encode('utf-8')) uni_value.encode('utf-8'))
@ -196,11 +198,11 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(head, name) resp = retry(head, name)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') self.assertEqual(resp.getheader('x-container-meta-test'), 'Value')
resp = retry(get, name) resp = retry(get, name)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') self.assertEqual(resp.getheader('x-container-meta-test'), 'Value')
resp = retry(delete, name) resp = retry(delete, name)
resp.read() resp.read()
@ -212,11 +214,11 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(head, name) resp = retry(head, name)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), None) self.assertEqual(resp.getheader('x-container-meta-test'), None)
resp = retry(get, name) resp = retry(get, name)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), None) self.assertEqual(resp.getheader('x-container-meta-test'), None)
resp = retry(delete, name) resp = retry(delete, name)
resp.read() resp.read()
@ -244,22 +246,22 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), None) self.assertEqual(resp.getheader('x-container-meta-test'), None)
resp = retry(get) resp = retry(get)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), None) self.assertEqual(resp.getheader('x-container-meta-test'), None)
resp = retry(post, 'Value') resp = retry(post, 'Value')
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(head) resp = retry(head)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') self.assertEqual(resp.getheader('x-container-meta-test'), 'Value')
resp = retry(get) resp = retry(get)
resp.read() resp.read()
self.assert_(resp.status in (200, 204), resp.status) self.assertIn(resp.status, (200, 204))
self.assertEqual(resp.getheader('x-container-meta-test'), 'Value') self.assertEqual(resp.getheader('x-container-meta-test'), 'Value')
def test_PUT_bad_metadata(self): def test_PUT_bad_metadata(self):
@ -319,7 +321,7 @@ class TestContainer(unittest.TestCase):
name = uuid4().hex name = uuid4().hex
headers = {} headers = {}
for x in xrange(self.max_meta_count): for x in range(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v' headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(put, name, headers) resp = retry(put, name, headers)
resp.read() resp.read()
@ -329,7 +331,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
name = uuid4().hex name = uuid4().hex
headers = {} headers = {}
for x in xrange(self.max_meta_count + 1): for x in range(self.max_meta_count + 1):
headers['X-Container-Meta-%d' % x] = 'v' headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(put, name, headers) resp = retry(put, name, headers)
resp.read() resp.read()
@ -368,12 +370,11 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 404) self.assertEqual(resp.status, 404)
def test_POST_bad_metadata(self): def test_POST_bad_metadata(self):
raise SkipTest('SOF constraints middleware enforces constraints.')
if tf.skip: if tf.skip:
raise SkipTest raise SkipTest
raise SkipTest('SOF constraints middleware enforces constraints.')
def post(url, token, parsed, conn, extra_headers): def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token} headers = {'X-Auth-Token': token}
headers.update(extra_headers) headers.update(extra_headers)
@ -415,13 +416,13 @@ class TestContainer(unittest.TestCase):
return check_response(conn) return check_response(conn)
headers = {} headers = {}
for x in xrange(self.max_meta_count): for x in range(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v' headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
headers = {} headers = {}
for x in xrange(self.max_meta_count + 1): for x in range(self.max_meta_count + 1):
headers['X-Container-Meta-%d' % x] = 'v' headers['X-Container-Meta-%d' % x] = 'v'
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
@ -452,8 +453,23 @@ class TestContainer(unittest.TestCase):
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
# this POST includes metadata size that is over limit
headers['X-Container-Meta-k'] = \ headers['X-Container-Meta-k'] = \
'v' * (self.max_meta_overall_size - size) 'x' * (self.max_meta_overall_size - size)
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 400)
# this POST would be ok and the aggregate backend metadata
# size is on the border
headers = {'X-Container-Meta-k':
'y' * (self.max_meta_overall_size - size - 1)}
resp = retry(post, headers)
resp.read()
self.assertEqual(resp.status, 204)
# this last POST would be ok by itself but takes the aggregate
# backend metadata size over limit
headers = {'X-Container-Meta-k':
'z' * (self.max_meta_overall_size - size)}
resp = retry(post, headers) resp = retry(post, headers)
resp.read() resp.read()
self.assertEqual(resp.status, 400) self.assertEqual(resp.status, 400)
@ -470,7 +486,7 @@ class TestContainer(unittest.TestCase):
resp = retry(get) resp = retry(get)
raise Exception('Should not have been able to GET') raise Exception('Should not have been able to GET')
except Exception as err: except Exception as err:
self.assert_(str(err).startswith('No result after '), err) self.assertTrue(str(err).startswith('No result after '), err)
def post(url, token, parsed, conn): def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '', conn.request('POST', parsed.path + '/' + self.name, '',
@ -497,7 +513,7 @@ class TestContainer(unittest.TestCase):
resp = retry(get) resp = retry(get)
raise Exception('Should not have been able to GET') raise Exception('Should not have been able to GET')
except Exception as err: except Exception as err:
self.assert_(str(err).startswith('No result after '), err) self.assertTrue(str(err).startswith('No result after '), err)
def test_cross_account_container(self): def test_cross_account_container(self):
if tf.skip or tf.skip2: if tf.skip or tf.skip2:
@ -715,7 +731,7 @@ class TestContainer(unittest.TestCase):
# cannot list containers # cannot list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-only access # grant read-only access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -728,23 +744,23 @@ class TestContainer(unittest.TestCase):
# read-only can list containers # read-only can list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.name in listing) self.assertIn(self.name, listing)
# read-only can not create containers # read-only can not create containers
new_container_name = str(uuid4()) new_container_name = str(uuid4())
resp = retry(put, new_container_name, use_account=3) resp = retry(put, new_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# but it can see newly created ones # but it can see newly created ones
resp = retry(put, new_container_name, use_account=1) resp = retry(put, new_container_name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(new_container_name in listing) self.assertIn(new_container_name, listing)
@requires_acls @requires_acls
def test_read_only_acl_metadata(self): def test_read_only_acl_metadata(self):
@ -774,13 +790,13 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# cannot see metadata # cannot see metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-only access # grant read-only access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -800,7 +816,7 @@ class TestContainer(unittest.TestCase):
# read-only can read container metadata # read-only can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
@requires_acls @requires_acls
@ -830,7 +846,7 @@ class TestContainer(unittest.TestCase):
# cannot list containers # cannot list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-write access # grant read-write access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -843,36 +859,36 @@ class TestContainer(unittest.TestCase):
# can list containers # can list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.name in listing) self.assertIn(self.name, listing)
# can create new containers # can create new containers
new_container_name = str(uuid4()) new_container_name = str(uuid4())
resp = retry(put, new_container_name, use_account=3) resp = retry(put, new_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(new_container_name in listing) self.assertIn(new_container_name, listing)
# can also delete them # can also delete them
resp = retry(delete, new_container_name, use_account=3) resp = retry(delete, new_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(new_container_name not in listing) self.assertNotIn(new_container_name, listing)
# even if they didn't create them # even if they didn't create them
empty_container_name = str(uuid4()) empty_container_name = str(uuid4())
resp = retry(put, empty_container_name, use_account=1) resp = retry(put, empty_container_name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(delete, empty_container_name, use_account=3) resp = retry(delete, empty_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
@requires_acls @requires_acls
def test_read_write_acl_metadata(self): def test_read_write_acl_metadata(self):
@ -902,13 +918,13 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# cannot see metadata # cannot see metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-write access # grant read-write access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -921,7 +937,7 @@ class TestContainer(unittest.TestCase):
# read-write can read container metadata # read-write can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# read-write can also write container metadata # read-write can also write container metadata
@ -929,20 +945,20 @@ class TestContainer(unittest.TestCase):
headers = {'x-container-meta-test': new_value} headers = {'x-container-meta-test': new_value}
resp = retry(post, self.name, headers=headers, use_account=3) resp = retry(post, self.name, headers=headers, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# and remove it # and remove it
headers = {'x-remove-container-meta-test': 'true'} headers = {'x-remove-container-meta-test': 'true'}
resp = retry(post, self.name, headers=headers, use_account=3) resp = retry(post, self.name, headers=headers, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None)
@requires_acls @requires_acls
@ -972,7 +988,7 @@ class TestContainer(unittest.TestCase):
# cannot list containers # cannot list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant admin access # grant admin access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -985,36 +1001,36 @@ class TestContainer(unittest.TestCase):
# can list containers # can list containers
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.name in listing) self.assertIn(self.name, listing)
# can create new containers # can create new containers
new_container_name = str(uuid4()) new_container_name = str(uuid4())
resp = retry(put, new_container_name, use_account=3) resp = retry(put, new_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(new_container_name in listing) self.assertIn(new_container_name, listing)
# can also delete them # can also delete them
resp = retry(delete, new_container_name, use_account=3) resp = retry(delete, new_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, use_account=3) resp = retry(get, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(new_container_name not in listing) self.assertNotIn(new_container_name, listing)
# even if they didn't create them # even if they didn't create them
empty_container_name = str(uuid4()) empty_container_name = str(uuid4())
resp = retry(put, empty_container_name, use_account=1) resp = retry(put, empty_container_name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(delete, empty_container_name, use_account=3) resp = retry(delete, empty_container_name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
@requires_acls @requires_acls
def test_admin_acl_metadata(self): def test_admin_acl_metadata(self):
@ -1044,13 +1060,13 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# cannot see metadata # cannot see metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant access # grant access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -1063,7 +1079,7 @@ class TestContainer(unittest.TestCase):
# can read container metadata # can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# can also write container metadata # can also write container metadata
@ -1071,20 +1087,20 @@ class TestContainer(unittest.TestCase):
headers = {'x-container-meta-test': new_value} headers = {'x-container-meta-test': new_value}
resp = retry(post, self.name, headers=headers, use_account=3) resp = retry(post, self.name, headers=headers, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# and remove it # and remove it
headers = {'x-remove-container-meta-test': 'true'} headers = {'x-remove-container-meta-test': 'true'}
resp = retry(post, self.name, headers=headers, use_account=3) resp = retry(post, self.name, headers=headers, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), None) self.assertEqual(resp.getheader('X-Container-Meta-Test'), None)
@requires_acls @requires_acls
@ -1118,7 +1134,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret')
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
@ -1133,7 +1149,7 @@ class TestContainer(unittest.TestCase):
# can read container metadata # can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# but not sync-key # but not sync-key
self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) self.assertEqual(resp.getheader('X-Container-Sync-Key'), None)
@ -1155,7 +1171,7 @@ class TestContainer(unittest.TestCase):
# can read container metadata # can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# but not sync-key # but not sync-key
self.assertEqual(resp.getheader('X-Container-Sync-Key'), None) self.assertEqual(resp.getheader('X-Container-Sync-Key'), None)
@ -1163,7 +1179,7 @@ class TestContainer(unittest.TestCase):
# sanity check sync-key w/ account1 # sanity check sync-key w/ account1
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret')
# and can write # and can write
@ -1177,7 +1193,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) # validate w/ account1 resp = retry(get, self.name, use_account=1) # validate w/ account1
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# but can not write sync-key # but can not write sync-key
self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret')
@ -1193,7 +1209,7 @@ class TestContainer(unittest.TestCase):
# admin can read container metadata # admin can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# and ALSO sync-key # and ALSO sync-key
self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret') self.assertEqual(resp.getheader('X-Container-Sync-Key'), 'secret')
@ -1206,7 +1222,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret) self.assertEqual(resp.getheader('X-Container-Sync-Key'), new_secret)
@requires_acls @requires_acls
@ -1241,7 +1257,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe')
self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe')
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
@ -1257,7 +1273,7 @@ class TestContainer(unittest.TestCase):
# can read container metadata # can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# but not container acl # but not container acl
self.assertEqual(resp.getheader('X-Container-Read'), None) self.assertEqual(resp.getheader('X-Container-Read'), None)
@ -1283,7 +1299,7 @@ class TestContainer(unittest.TestCase):
# can read container metadata # can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), value)
# but not container acl # but not container acl
self.assertEqual(resp.getheader('X-Container-Read'), None) self.assertEqual(resp.getheader('X-Container-Read'), None)
@ -1292,7 +1308,7 @@ class TestContainer(unittest.TestCase):
# sanity check container acls with account1 # sanity check container acls with account1
resp = retry(get, self.name, use_account=1) resp = retry(get, self.name, use_account=1)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe')
self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Write'), 'jdoe')
@ -1308,7 +1324,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=1) # validate w/ account1 resp = retry(get, self.name, use_account=1) # validate w/ account1
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# but can not write container acls # but can not write container acls
self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe')
@ -1325,7 +1341,7 @@ class TestContainer(unittest.TestCase):
# admin can read container metadata # admin can read container metadata
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value) self.assertEqual(resp.getheader('X-Container-Meta-Test'), new_value)
# and ALSO container acls # and ALSO container acls
self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe') self.assertEqual(resp.getheader('X-Container-Read'), 'jdoe')
@ -1341,7 +1357,7 @@ class TestContainer(unittest.TestCase):
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
resp = retry(get, self.name, use_account=3) resp = retry(get, self.name, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
self.assertEqual(resp.getheader('X-Container-Read'), '.r:*') self.assertEqual(resp.getheader('X-Container-Read'), '.r:*')
def test_long_name_content_type(self): def test_long_name_content_type(self):
@ -1384,8 +1400,11 @@ class TestContainer(unittest.TestCase):
raise SkipTest() raise SkipTest()
def put(url, token, parsed, conn): def put(url, token, parsed, conn):
# using the empty storage policy header value here to ensure
# that the default policy is chosen in case policy_specified is set
# see __init__.py for details on policy_specified
conn.request('PUT', parsed.path + '/' + self.container, '', conn.request('PUT', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token}) {'X-Auth-Token': token, 'X-Storage-Policy': ''})
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
resp.read() resp.read()
@ -1398,7 +1417,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), self.assertEqual(headers.get('x-storage-policy'),
default_policy['name']) default_policy['name'])
def test_error_invalid_storage_policy_name(self): def test_error_invalid_storage_policy_name(self):
@ -1436,7 +1455,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), self.assertEqual(headers.get('x-storage-policy'),
policy['name']) policy['name'])
# and test recreate with-out specifying Storage Policy # and test recreate with-out specifying Storage Policy
@ -1447,7 +1466,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), self.assertEqual(headers.get('x-storage-policy'),
policy['name']) policy['name'])
# delete it # delete it
@ -1463,7 +1482,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), None) self.assertEqual(headers.get('x-storage-policy'), None)
@requires_policies @requires_policies
def test_conflict_change_storage_policy_with_put(self): def test_conflict_change_storage_policy_with_put(self):
@ -1493,7 +1512,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), self.assertEqual(headers.get('x-storage-policy'),
policy['name']) policy['name'])
@requires_policies @requires_policies
@ -1530,7 +1549,7 @@ class TestContainer(unittest.TestCase):
resp = retry(head) resp = retry(head)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('x-storage-policy'), self.assertEqual(headers.get('x-storage-policy'),
policy['name']) policy['name'])
@ -1578,7 +1597,7 @@ class BaseTestContainerACLs(unittest.TestCase):
while True: while True:
resp = retry(get, use_account=self.account) resp = retry(get, use_account=self.account)
body = resp.read() body = resp.read()
self.assert_(resp.status // 100 == 2, resp.status) self.assertTrue(resp.status // 100 == 2, resp.status)
objs = json.loads(body) objs = json.loads(body)
if not objs: if not objs:
break break

152
test/functional/test_object.py
View File

@ -15,11 +15,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import json
import unittest import unittest
from nose import SkipTest from nose import SkipTest
from uuid import uuid4 from uuid import uuid4
from swift.common.utils import json from six.moves import range
from test.functional import check_response, retry, requires_acls, \ from test.functional import check_response, retry, requires_acls, \
requires_policies requires_policies
@ -88,7 +89,7 @@ class TestObject(unittest.TestCase):
body = resp.read() body = resp.read()
if resp.status == 404: if resp.status == 404:
break break
self.assert_(resp.status // 100 == 2, resp.status) self.assertTrue(resp.status // 100 == 2, resp.status)
objs = json.loads(body) objs = json.loads(body)
if not objs: if not objs:
break break
@ -106,7 +107,7 @@ class TestObject(unittest.TestCase):
for container in self.containers: for container in self.containers:
resp = retry(delete, container) resp = retry(delete, container)
resp.read() resp.read()
self.assert_(resp.status in (204, 404)) self.assertIn(resp.status, (204, 404))
def test_if_none_match(self): def test_if_none_match(self):
def put(url, token, parsed, conn): def put(url, token, parsed, conn):
@ -118,10 +119,10 @@ class TestObject(unittest.TestCase):
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
resp.read() resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
resp = retry(put) resp = retry(put)
resp.read() resp.read()
self.assertEquals(resp.status, 412) self.assertEqual(resp.status, 412)
def put(url, token, parsed, conn): def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % ( conn.request('PUT', '%s/%s/%s' % (
@ -132,7 +133,7 @@ class TestObject(unittest.TestCase):
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
resp.read() resp.read()
self.assertEquals(resp.status, 400) self.assertEqual(resp.status, 400)
def test_non_integer_x_delete_after(self): def test_non_integer_x_delete_after(self):
def put(url, token, parsed, conn): def put(url, token, parsed, conn):
@ -144,7 +145,7 @@ class TestObject(unittest.TestCase):
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 400) self.assertEqual(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-After') self.assertEqual(body, 'Non-integer X-Delete-After')
def test_non_integer_x_delete_at(self): def test_non_integer_x_delete_at(self):
@ -157,7 +158,7 @@ class TestObject(unittest.TestCase):
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 400) self.assertEqual(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-At') self.assertEqual(body, 'Non-integer X-Delete-At')
def test_x_delete_at_in_the_past(self): def test_x_delete_at_in_the_past(self):
@ -170,7 +171,7 @@ class TestObject(unittest.TestCase):
return check_response(conn) return check_response(conn)
resp = retry(put) resp = retry(put)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 400) self.assertEqual(resp.status, 400)
self.assertEqual(body, 'X-Delete-At in past') self.assertEqual(body, 'X-Delete-At in past')
def test_copy_object(self): def test_copy_object(self):
@ -242,6 +243,23 @@ class TestObject(unittest.TestCase):
self.assertEqual(resp.status, 200) self.assertEqual(resp.status, 200)
self.assertEqual(dest_contents, source_contents) self.assertEqual(dest_contents, source_contents)
# copy source to dest with COPY and range
def copy(url, token, parsed, conn):
conn.request('COPY', '%s/%s' % (parsed.path, source), '',
{'X-Auth-Token': token,
'Destination': dest,
'Range': 'bytes=1-2'})
return check_response(conn)
resp = retry(copy)
resp.read()
self.assertEqual(resp.status, 201)
# contents of dest should be the same as source
resp = retry(get_dest)
dest_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(dest_contents, source_contents[1:3])
# delete the copy # delete the copy
resp = retry(delete) resp = retry(delete)
resp.read() resp.read()
@ -369,7 +387,7 @@ class TestObject(unittest.TestCase):
resp = retry(get) resp = retry(get)
raise Exception('Should not have been able to GET') raise Exception('Should not have been able to GET')
except Exception as err: except Exception as err:
self.assert_(str(err).startswith('No result after ')) self.assertTrue(str(err).startswith('No result after '))
def post(url, token, parsed, conn): def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.container, '', conn.request('POST', parsed.path + '/' + self.container, '',
@ -394,7 +412,7 @@ class TestObject(unittest.TestCase):
resp = retry(get) resp = retry(get)
raise Exception('Should not have been able to GET') raise Exception('Should not have been able to GET')
except Exception as err: except Exception as err:
self.assert_(str(err).startswith('No result after ')) self.assertTrue(str(err).startswith('No result after '))
def test_private_object(self): def test_private_object(self):
if tf.skip or tf.skip3: if tf.skip or tf.skip3:
@ -525,12 +543,12 @@ class TestObject(unittest.TestCase):
# cannot list objects # cannot list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# cannot get object # cannot get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-only access # grant read-only access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -543,32 +561,32 @@ class TestObject(unittest.TestCase):
# can list objects # can list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.obj in listing) self.assertIn(self.obj, listing)
# can get object # can get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assertEquals(body, 'test') self.assertEqual(body, 'test')
# can not put an object # can not put an object
obj_name = str(uuid4()) obj_name = str(uuid4())
resp = retry(put, obj_name, use_account=3) resp = retry(put, obj_name, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# can not delete an object # can not delete an object
resp = retry(delete, self.obj, use_account=3) resp = retry(delete, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# sanity with account1 # sanity with account1
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(obj_name not in listing) self.assertNotIn(obj_name, listing)
self.assert_(self.obj in listing) self.assertIn(self.obj, listing)
@requires_acls @requires_acls
def test_read_write(self): def test_read_write(self):
@ -606,12 +624,12 @@ class TestObject(unittest.TestCase):
# cannot list objects # cannot list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# cannot get object # cannot get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant read-write access # grant read-write access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -624,32 +642,32 @@ class TestObject(unittest.TestCase):
# can list objects # can list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.obj in listing) self.assertIn(self.obj, listing)
# can get object # can get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assertEquals(body, 'test') self.assertEqual(body, 'test')
# can put an object # can put an object
obj_name = str(uuid4()) obj_name = str(uuid4())
resp = retry(put, obj_name, use_account=3) resp = retry(put, obj_name, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
# can delete an object # can delete an object
resp = retry(delete, self.obj, use_account=3) resp = retry(delete, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
# sanity with account1 # sanity with account1
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(obj_name in listing) self.assertIn(obj_name, listing)
self.assert_(self.obj not in listing) self.assertNotIn(self.obj, listing)
@requires_acls @requires_acls
def test_admin(self): def test_admin(self):
@ -687,12 +705,12 @@ class TestObject(unittest.TestCase):
# cannot list objects # cannot list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# cannot get object # cannot get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
resp.read() resp.read()
self.assertEquals(resp.status, 403) self.assertEqual(resp.status, 403)
# grant admin access # grant admin access
acl_user = tf.swift_test_user[2] acl_user = tf.swift_test_user[2]
@ -705,32 +723,32 @@ class TestObject(unittest.TestCase):
# can list objects # can list objects
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(self.obj in listing) self.assertIn(self.obj, listing)
# can get object # can get object
resp = retry(get, self.obj, use_account=3) resp = retry(get, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assertEquals(body, 'test') self.assertEqual(body, 'test')
# can put an object # can put an object
obj_name = str(uuid4()) obj_name = str(uuid4())
resp = retry(put, obj_name, use_account=3) resp = retry(put, obj_name, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 201) self.assertEqual(resp.status, 201)
# can delete an object # can delete an object
resp = retry(delete, self.obj, use_account=3) resp = retry(delete, self.obj, use_account=3)
body = resp.read() body = resp.read()
self.assertEquals(resp.status, 204) self.assertEqual(resp.status, 204)
# sanity with account1 # sanity with account1
resp = retry(get_listing, use_account=3) resp = retry(get_listing, use_account=3)
listing = resp.read() listing = resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
self.assert_(obj_name in listing) self.assertIn(obj_name, listing)
self.assert_(self.obj not in listing) self.assertNotIn(self.obj, listing)
def test_manifest(self): def test_manifest(self):
if tf.skip: if tf.skip:
@ -746,7 +764,7 @@ class TestObject(unittest.TestCase):
parsed.path, self.container, str(objnum)), segments1[objnum], parsed.path, self.container, str(objnum)), segments1[objnum],
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments1)): for objnum in range(len(segments1)):
resp = retry(put, objnum) resp = retry(put, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 201) self.assertEqual(resp.status, 201)
@ -809,7 +827,7 @@ class TestObject(unittest.TestCase):
parsed.path, self.container, str(objnum)), segments2[objnum], parsed.path, self.container, str(objnum)), segments2[objnum],
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments2)): for objnum in range(len(segments2)):
resp = retry(put, objnum) resp = retry(put, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 201) self.assertEqual(resp.status, 201)
@ -891,7 +909,7 @@ class TestObject(unittest.TestCase):
parsed.path, acontainer, str(objnum)), segments3[objnum], parsed.path, acontainer, str(objnum)), segments3[objnum],
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments3)): for objnum in range(len(segments3)):
resp = retry(put, objnum) resp = retry(put, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 201) self.assertEqual(resp.status, 201)
@ -966,7 +984,7 @@ class TestObject(unittest.TestCase):
parsed.path, acontainer, str(objnum)), '', parsed.path, acontainer, str(objnum)), '',
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments3)): for objnum in range(len(segments3)):
resp = retry(delete, objnum) resp = retry(delete, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
@ -977,7 +995,7 @@ class TestObject(unittest.TestCase):
parsed.path, self.container, str(objnum)), '', parsed.path, self.container, str(objnum)), '',
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments2)): for objnum in range(len(segments2)):
resp = retry(delete, objnum) resp = retry(delete, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
@ -988,7 +1006,7 @@ class TestObject(unittest.TestCase):
parsed.path, self.container, str(objnum)), '', parsed.path, self.container, str(objnum)), '',
{'X-Auth-Token': token}) {'X-Auth-Token': token})
return check_response(conn) return check_response(conn)
for objnum in xrange(len(segments1)): for objnum in range(len(segments1)):
resp = retry(delete, objnum) resp = retry(delete, objnum)
resp.read() resp.read()
self.assertEqual(resp.status, 204) self.assertEqual(resp.status, 204)
@ -1095,77 +1113,77 @@ class TestObject(unittest.TestCase):
resp = retry(put_cors_cont, '*') resp = retry(put_cors_cont, '*')
resp.read() resp.read()
self.assertEquals(resp.status // 100, 2) self.assertEqual(resp.status // 100, 2)
resp = retry(put_obj, 'cat') resp = retry(put_obj, 'cat')
resp.read() resp.read()
self.assertEquals(resp.status // 100, 2) self.assertEqual(resp.status // 100, 2)
resp = retry(check_cors, resp = retry(check_cors,
'OPTIONS', 'cat', {'Origin': 'http://m.com'}) 'OPTIONS', 'cat', {'Origin': 'http://m.com'})
self.assertEquals(resp.status, 401) self.assertEqual(resp.status, 401)
resp = retry(check_cors, resp = retry(check_cors,
'OPTIONS', 'cat', 'OPTIONS', 'cat',
{'Origin': 'http://m.com', {'Origin': 'http://m.com',
'Access-Control-Request-Method': 'GET'}) 'Access-Control-Request-Method': 'GET'})
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
resp.read() resp.read()
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('access-control-allow-origin'), self.assertEqual(headers.get('access-control-allow-origin'),
'*') '*')
resp = retry(check_cors, resp = retry(check_cors,
'GET', 'cat', {'Origin': 'http://m.com'}) 'GET', 'cat', {'Origin': 'http://m.com'})
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('access-control-allow-origin'), self.assertEqual(headers.get('access-control-allow-origin'),
'*') '*')
resp = retry(check_cors, resp = retry(check_cors,
'GET', 'cat', {'Origin': 'http://m.com', 'GET', 'cat', {'Origin': 'http://m.com',
'X-Web-Mode': 'True'}) 'X-Web-Mode': 'True'})
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('access-control-allow-origin'), self.assertEqual(headers.get('access-control-allow-origin'),
'*') '*')
#################### ####################
resp = retry(put_cors_cont, 'http://secret.com') resp = retry(put_cors_cont, 'http://secret.com')
resp.read() resp.read()
self.assertEquals(resp.status // 100, 2) self.assertEqual(resp.status // 100, 2)
resp = retry(check_cors, resp = retry(check_cors,
'OPTIONS', 'cat', 'OPTIONS', 'cat',
{'Origin': 'http://m.com', {'Origin': 'http://m.com',
'Access-Control-Request-Method': 'GET'}) 'Access-Control-Request-Method': 'GET'})
resp.read() resp.read()
self.assertEquals(resp.status, 401) self.assertEqual(resp.status, 401)
if strict_cors: if strict_cors:
resp = retry(check_cors, resp = retry(check_cors,
'GET', 'cat', {'Origin': 'http://m.com'}) 'GET', 'cat', {'Origin': 'http://m.com'})
resp.read() resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertTrue('access-control-allow-origin' not in headers) self.assertNotIn('access-control-allow-origin', headers)
resp = retry(check_cors, resp = retry(check_cors,
'GET', 'cat', {'Origin': 'http://secret.com'}) 'GET', 'cat', {'Origin': 'http://secret.com'})
resp.read() resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('access-control-allow-origin'), self.assertEqual(headers.get('access-control-allow-origin'),
'http://secret.com') 'http://secret.com')
else: else:
resp = retry(check_cors, resp = retry(check_cors,
'GET', 'cat', {'Origin': 'http://m.com'}) 'GET', 'cat', {'Origin': 'http://m.com'})
resp.read() resp.read()
self.assertEquals(resp.status, 200) self.assertEqual(resp.status, 200)
headers = dict((k.lower(), v) for k, v in resp.getheaders()) headers = dict((k.lower(), v) for k, v in resp.getheaders())
self.assertEquals(headers.get('access-control-allow-origin'), self.assertEqual(headers.get('access-control-allow-origin'),
'http://m.com') 'http://m.com')
@requires_policies @requires_policies

1067
test/functional/tests.py
View File

File diff suppressed because it is too large Load Diff

157
test/unit/__init__.py
View File

@ -15,10 +15,12 @@
""" Swift tests """ """ Swift tests """
from __future__ import print_function
import os import os
import copy import copy
import logging import logging
import errno import errno
from six.moves import range
import sys import sys
from contextlib import contextmanager, closing from contextlib import contextmanager, closing
from collections import defaultdict, Iterable from collections import defaultdict, Iterable
@ -30,17 +32,18 @@ import eventlet
from eventlet.green import socket from eventlet.green import socket
from tempfile import mkdtemp from tempfile import mkdtemp
from shutil import rmtree from shutil import rmtree
from swift.common.utils import Timestamp from swift.common.utils import Timestamp, NOTICE
from test import get_config from test import get_config
from swift.common import swob, utils from swift.common import swob, utils
from swift.common.ring import Ring, RingData from swift.common.ring import Ring, RingData
from hashlib import md5 from hashlib import md5
import logging.handlers import logging.handlers
from httplib import HTTPException
from six.moves.http_client import HTTPException
from swift.common import storage_policy from swift.common import storage_policy
from swift.common.storage_policy import StoragePolicy, ECStoragePolicy from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
import functools import functools
import cPickle as pickle import six.moves.cPickle as pickle
from gzip import GzipFile from gzip import GzipFile
import mock as mocklib import mock as mocklib
import inspect import inspect
@ -226,9 +229,7 @@ class FakeRing(Ring):
return [dict(node, index=i) for i, node in enumerate(list(self._devs))] return [dict(node, index=i) for i, node in enumerate(list(self._devs))]
def get_more_nodes(self, part): def get_more_nodes(self, part):
# replicas^2 is the true cap for x in range(self.replicas, (self.replicas + self.max_more_nodes)):
for x in xrange(self.replicas, min(self.replicas + self.max_more_nodes,
self.replicas * self.replicas)):
yield {'ip': '10.0.0.%s' % x, yield {'ip': '10.0.0.%s' % x,
'replication_ip': '10.0.0.%s' % x, 'replication_ip': '10.0.0.%s' % x,
'port': self._base_port + x, 'port': self._base_port + x,
@ -478,8 +479,18 @@ class FakeLogger(logging.Logger, object):
logging.INFO: 'info', logging.INFO: 'info',
logging.DEBUG: 'debug', logging.DEBUG: 'debug',
logging.CRITICAL: 'critical', logging.CRITICAL: 'critical',
NOTICE: 'notice',
} }
def notice(self, msg, *args, **kwargs):
"""
Convenience function for syslog priority LOG_NOTICE. The python
logging lvl is set to 25, just above info. SysLogHandler is
monkey patched to map this log lvl to the LOG_NOTICE syslog
priority.
"""
self.log(NOTICE, msg, *args, **kwargs)
def _log(self, level, msg, *args, **kwargs): def _log(self, level, msg, *args, **kwargs):
store_name = self.store_in[level] store_name = self.store_in[level]
cargs = [msg] cargs = [msg]
@ -495,7 +506,9 @@ class FakeLogger(logging.Logger, object):
def _clear(self): def _clear(self):
self.log_dict = defaultdict(list) self.log_dict = defaultdict(list)
self.lines_dict = {'critical': [], 'error': [], 'info': [], self.lines_dict = {'critical': [], 'error': [], 'info': [],
'warning': [], 'debug': []} 'warning': [], 'debug': [], 'notice': []}
clear = _clear # this is a public interface
def get_lines_for_level(self, level): def get_lines_for_level(self, level):
if level not in self.lines_dict: if level not in self.lines_dict:
@ -560,8 +573,8 @@ class FakeLogger(logging.Logger, object):
try: try:
line = record.getMessage() line = record.getMessage()
except TypeError: except TypeError:
print 'WARNING: unable to format log message %r %% %r' % ( print('WARNING: unable to format log message %r %% %r' % (
record.msg, record.args) record.msg, record.args))
raise raise
self.lines_dict[record.levelname.lower()].append(line) self.lines_dict[record.levelname.lower()].append(line)
@ -585,7 +598,7 @@ class DebugLogger(FakeLogger):
def handle(self, record): def handle(self, record):
self._handle(record) self._handle(record)
print self.formatter.format(record) print(self.formatter.format(record))
class DebugLogAdapter(utils.LogAdapter): class DebugLogAdapter(utils.LogAdapter):
@ -704,6 +717,74 @@ def mock(update):
delattr(module, attr) delattr(module, attr)
class FakeStatus(object):
"""
This will work with our fake_http_connect, if you hand in one of these
instead of a status int or status int tuple to the "codes" iter you can
add some eventlet sleep to the expect and response stages of the
connection.
"""
def __init__(self, status, expect_sleep=None, response_sleep=None):
"""
:param status: the response status int, or a tuple of
([expect_status, ...], response_status)
:param expect_sleep: float, time to eventlet sleep during expect, can
be a iter of floats
:param response_sleep: float, time to eventlet sleep during response
"""
# connect exception
if isinstance(status, (Exception, eventlet.Timeout)):
raise status
if isinstance(status, tuple):
self.expect_status = list(status[:-1])
self.status = status[-1]
self.explicit_expect_list = True
else:
self.expect_status, self.status = ([], status)
self.explicit_expect_list = False
if not self.expect_status:
# when a swift backend service returns a status before reading
# from the body (mostly an error response) eventlet.wsgi will
# respond with that status line immediately instead of 100
# Continue, even if the client sent the Expect 100 header.
# BufferedHttp and the proxy both see these error statuses
# when they call getexpect, so our FakeConn tries to act like
# our backend services and return certain types of responses
# as expect statuses just like a real backend server would do.
if self.status in (507, 412, 409):
self.expect_status = [status]
else:
self.expect_status = [100, 100]
# setup sleep attributes
if not isinstance(expect_sleep, (list, tuple)):
expect_sleep = [expect_sleep] * len(self.expect_status)
self.expect_sleep_list = list(expect_sleep)
while len(self.expect_sleep_list) < len(self.expect_status):
self.expect_sleep_list.append(None)
self.response_sleep = response_sleep
def get_response_status(self):
if self.response_sleep is not None:
eventlet.sleep(self.response_sleep)
if self.expect_status and self.explicit_expect_list:
raise Exception('Test did not consume all fake '
'expect status: %r' % (self.expect_status,))
if isinstance(self.status, (Exception, eventlet.Timeout)):
raise self.status
return self.status
def get_expect_status(self):
expect_sleep = self.expect_sleep_list.pop(0)
if expect_sleep is not None:
eventlet.sleep(expect_sleep)
expect_status = self.expect_status.pop(0)
if isinstance(expect_status, (Exception, eventlet.Timeout)):
raise expect_status
return expect_status
class SlowBody(object): class SlowBody(object):
""" """
This will work with our fake_http_connect, if you hand in these This will work with our fake_http_connect, if you hand in these
@ -741,29 +822,9 @@ def fake_http_connect(*code_iter, **kwargs):
def __init__(self, status, etag=None, body='', timestamp='1', def __init__(self, status, etag=None, body='', timestamp='1',
headers=None, expect_headers=None, connection_id=None, headers=None, expect_headers=None, connection_id=None,
give_send=None): give_send=None):
# connect exception if not isinstance(status, FakeStatus):
if isinstance(status, (Exception, eventlet.Timeout)): status = FakeStatus(status)
raise status self._status = status
if isinstance(status, tuple):
self.expect_status = list(status[:-1])
self.status = status[-1]
self.explicit_expect_list = True
else:
self.expect_status, self.status = ([], status)
self.explicit_expect_list = False
if not self.expect_status:
# when a swift backend service returns a status before reading
# from the body (mostly an error response) eventlet.wsgi will
# respond with that status line immediately instead of 100
# Continue, even if the client sent the Expect 100 header.
# BufferedHttp and the proxy both see these error statuses
# when they call getexpect, so our FakeConn tries to act like
# our backend services and return certain types of responses
# as expect statuses just like a real backend server would do.
if self.status in (507, 412, 409):
self.expect_status = [status]
else:
self.expect_status = [100, 100]
self.reason = 'Fake' self.reason = 'Fake'
self.host = '1.2.3.4' self.host = '1.2.3.4'
self.port = '1234' self.port = '1234'
@ -785,11 +846,6 @@ def fake_http_connect(*code_iter, **kwargs):
eventlet.sleep() eventlet.sleep()
def getresponse(self): def getresponse(self):
if self.expect_status and self.explicit_expect_list:
raise Exception('Test did not consume all fake '
'expect status: %r' % (self.expect_status,))
if isinstance(self.status, (Exception, eventlet.Timeout)):
raise self.status
exc = kwargs.get('raise_exc') exc = kwargs.get('raise_exc')
if exc: if exc:
if isinstance(exc, (Exception, eventlet.Timeout)): if isinstance(exc, (Exception, eventlet.Timeout)):
@ -797,16 +853,19 @@ def fake_http_connect(*code_iter, **kwargs):
raise Exception('test') raise Exception('test')
if kwargs.get('raise_timeout_exc'): if kwargs.get('raise_timeout_exc'):
raise eventlet.Timeout() raise eventlet.Timeout()
self.status = self._status.get_response_status()
return self return self
def getexpect(self): def getexpect(self):
expect_status = self.expect_status.pop(0) expect_status = self._status.get_expect_status()
if isinstance(self.expect_status, (Exception, eventlet.Timeout)):
raise self.expect_status
headers = dict(self.expect_headers) headers = dict(self.expect_headers)
if expect_status == 409: if expect_status == 409:
headers['X-Backend-Timestamp'] = self.timestamp headers['X-Backend-Timestamp'] = self.timestamp
return FakeConn(expect_status, headers=headers) response = FakeConn(expect_status,
timestamp=self.timestamp,
headers=headers)
response.status = expect_status
return response
def getheaders(self): def getheaders(self):
etag = self.etag etag = self.etag
@ -834,7 +893,7 @@ def fake_http_connect(*code_iter, **kwargs):
# when timestamp is None, HeaderKeyDict raises KeyError # when timestamp is None, HeaderKeyDict raises KeyError
headers.pop('x-timestamp', None) headers.pop('x-timestamp', None)
try: try:
if container_ts_iter.next() is False: if next(container_ts_iter) is False:
headers['x-container-timestamp'] = '1' headers['x-container-timestamp'] = '1'
except StopIteration: except StopIteration:
pass pass
@ -911,24 +970,24 @@ def fake_http_connect(*code_iter, **kwargs):
kwargs['give_content_type'](args[6]['Content-Type']) kwargs['give_content_type'](args[6]['Content-Type'])
else: else:
kwargs['give_content_type']('') kwargs['give_content_type']('')
i, status = conn_id_and_code_iter.next() i, status = next(conn_id_and_code_iter)
if 'give_connect' in kwargs: if 'give_connect' in kwargs:
give_conn_fn = kwargs['give_connect'] give_conn_fn = kwargs['give_connect']
argspec = inspect.getargspec(give_conn_fn) argspec = inspect.getargspec(give_conn_fn)
if argspec.keywords or 'connection_id' in argspec.args: if argspec.keywords or 'connection_id' in argspec.args:
ckwargs['connection_id'] = i ckwargs['connection_id'] = i
give_conn_fn(*args, **ckwargs) give_conn_fn(*args, **ckwargs)
etag = etag_iter.next() etag = next(etag_iter)
headers = headers_iter.next() headers = next(headers_iter)
expect_headers = expect_headers_iter.next() expect_headers = next(expect_headers_iter)
timestamp = timestamps_iter.next() timestamp = next(timestamps_iter)
if status <= 0: if status <= 0:
raise HTTPException() raise HTTPException()
if body_iter is None: if body_iter is None:
body = static_body or '' body = static_body or ''
else: else:
body = body_iter.next() body = next(body_iter)
return FakeConn(status, etag, body=body, timestamp=timestamp, return FakeConn(status, etag, body=body, timestamp=timestamp,
headers=headers, expect_headers=expect_headers, headers=headers, expect_headers=expect_headers,
connection_id=i, give_send=kwargs.get('give_send')) connection_id=i, give_send=kwargs.get('give_send'))

4
test/unit/obj/test_diskfile.py
View File

@ -207,9 +207,6 @@ class TestDiskFile(unittest.TestCase):
assert not gdf._is_dir assert not gdf._is_dir
assert gdf._fd is not None assert gdf._fd is not None
assert gdf._metadata == exp_md assert gdf._metadata == exp_md
self.assertRaises(DiskFileNotOpen, gdf.get_metadata)
self.assertRaises(DiskFileNotOpen, gdf.reader)
self.assertRaises(DiskFileNotOpen, gdf.__enter__)
def test_open_and_close(self): def test_open_and_close(self):
mock_close = Mock() mock_close = Mock()
@ -480,7 +477,6 @@ class TestDiskFile(unittest.TestCase):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z") gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'} md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.write_metadata(md.copy()) gdf.write_metadata(md.copy())
self.assertEqual(None, gdf._metadata)
fmd = _metadata[_mapit(the_dir)] fmd = _metadata[_mapit(the_dir)]
md.update({'X-Object-Type': 'file', 'X-Type': 'Object'}) md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
self.assertTrue(fmd['a'], md['a']) self.assertTrue(fmd['a'], md['a'])

32
tox.ini
View File

@ -1,5 +1,5 @@
[tox] [tox]
envlist = py26,py27,pep8,functest,functest-ci envlist = py27,pep8,functest
minversion = 1.6 minversion = 1.6
skipsdist = True skipsdist = True
@ -15,11 +15,12 @@ deps =
# Note: pip supports installing from git repos. # Note: pip supports installing from git repos.
# https://pip.pypa.io/en/latest/reference/pip_install.html#git # https://pip.pypa.io/en/latest/reference/pip_install.html#git
# Example: git+https://github.com/openstack/swift.git@2.0.0 # Example: git+https://github.com/openstack/swift.git@2.0.0
https://launchpad.net/swift/kilo/2.3.0/+download/swift-2.3.0.tar.gz https://launchpad.net/swift/liberty/2.5.0/+download/swift-2.5.0.tar.gz
PyECLib==1.0.7 PyECLib==1.0.7
-r{toxinidir}/test-requirements.txt -r{toxinidir}/test-requirements.txt
changedir = {toxinidir}/test/unit changedir = {toxinidir}/test/unit
commands = nosetests -v {posargs} commands = nosetests -v {posargs}
passenv = SWIFT_* *_proxy
[testenv:cover] [testenv:cover]
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
@ -28,9 +29,6 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_COVER_HTML=1 NOSE_COVER_HTML=1
NOSE_COVER_HTML_DIR={toxinidir}/cover NOSE_COVER_HTML_DIR={toxinidir}/cover
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:functest] [testenv:functest]
changedir = {toxinidir} changedir = {toxinidir}
commands = bash ./.functests -q commands = bash ./.functests -q
@ -46,14 +44,20 @@ changedir = {toxinidir}
commands = {posargs} commands = {posargs}
[flake8] [flake8]
# it's not a bug that we aren't using all of hacking # it's not a bug that we aren't using all of hacking, ignore:
# H102 -> apache2 license exists # F812: list comprehension redefines ...
# H103 -> license is apache # H101: Use TODO(NAME)
# H201 -> no bare excepts (unless marked with " # noqa") # H202: assertRaises Exception too broad
# H231 -> Check for except statements to be Python 3.x compatible # H233: Python 3.x incompatible use of print operator
# H501 -> don't use locals() for str formatting # H234: assertEquals is deprecated, use assertEqual
# H903 -> \n not \r\n # H301: one import per line
ignore = H # H306: imports not in alphabetical order (time, os)
select = F,E,W,H102,H103,H201,H231,H501,H903 # H401: docstring should not start with a space
# H403: multi line docstrings should end on a new line
# H404: multi line docstring should start without a leading new line
# H405: multi line docstring summary not separated with an empty line
# H501: Do not use self.__dict__ for string formatting
# H703: Multiple positional placeholders
ignore = F812,H101,H202,H233,H234,H301,H306,H401,H403,H404,H405,H501,H703
exclude = .venv,.tox,dist,doc,*egg,test exclude = .venv,.tox,dist,doc,*egg,test
show-source = True show-source = True