Rebase to Swift 2.2.1 release

NOTE:
The previous rebase was to Swift 2.1.0 and this rebase is to
Swift 2.2.1 (first release in kilo series). There was a
Swift 2.2.0 (last release in juno series) release in between.

Change-Id: Ibce2e299935e165db89a91a6fe8c4c5c027db098
Signed-off-by: Prashanth Pai <ppai@redhat.com>
This commit is contained in:
Prashanth Pai 2015-01-15 12:31:55 +05:30
parent 486668b880
commit aa1bfb3e67
12 changed files with 855 additions and 106 deletions

View File

@ -1,3 +1,7 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
dnspython>=1.9.4
eventlet>=0.9.15
greenlet>=0.3.1

View File

@ -43,6 +43,6 @@ class PkgInfo(object):
# Change the Package version here
_pkginfo = PkgInfo('2.1.0', '0', 'swiftonfile', False)
_pkginfo = PkgInfo('2.2.1', '0', 'swiftonfile', False)
__version__ = _pkginfo.pretty_version
__canonical_version__ = _pkginfo.canonical_version

View File

@ -1,3 +1,7 @@
# The order of packages is significant, because pip processes them in the order
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
# Hacking already pins down pep8, pyflakes and flake8
hacking>=0.8.0,<0.9
coverage

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import os
import sys
import pickle
@ -30,6 +31,7 @@ from contextlib import closing
from gzip import GzipFile
from shutil import rmtree
from tempfile import mkdtemp
from swift.common.middleware.memcache import MemcacheMiddleware
from test import get_config
from test.functional.swift_test_client import Account, Connection, \
@ -40,15 +42,12 @@ from test.functional.swift_test_client import Account, Connection, \
from test.unit import debug_logger, FakeMemcache
from swift.common import constraints, utils, ring, storage_policy
from swift.common.wsgi import monkey_patch_mimetools
from swift.common.middleware import catch_errors, gatekeeper, healthcheck, \
proxy_logging, container_sync, bulk, tempurl, slo, dlo, ratelimit, \
tempauth, container_quotas, account_quotas
from swift.common.ring import Ring
from swift.common.wsgi import monkey_patch_mimetools, loadapp
from swift.common.utils import config_true_value
from swift.proxy import server as proxy_server
from swift.account import server as account_server
from swift.container import server as container_server
from swift.obj import server as object_server
from swift.obj import server as object_server, mem_server as mem_object_server
import swift.proxy.controllers.obj
# In order to get the proper blocking behavior of sockets without using
@ -83,10 +82,13 @@ normalized_urls = None
# If no config was read, we will fall back to old school env vars
swift_test_auth_version = None
swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None]
swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None]
swift_test_tenant = ['', '', '']
swift_test_perm = ['', '', '']
swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None, '']
swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None, '']
swift_test_tenant = ['', '', '', '']
swift_test_perm = ['', '', '', '']
swift_test_domain = ['', '', '', '']
swift_test_user_id = ['', '', '', '']
swift_test_tenant_id = ['', '', '', '']
skip, skip2, skip3 = False, False, False
@ -100,25 +102,16 @@ in_process = False
_testdir = _test_servers = _test_sockets = _test_coros = None
class FakeMemcacheMiddleware(object):
class FakeMemcacheMiddleware(MemcacheMiddleware):
"""
Caching middleware that fakes out caching in swift.
Caching middleware that fakes out caching in swift if memcached
does not appear to be running.
"""
def __init__(self, app, conf):
self.app = app
super(FakeMemcacheMiddleware, self).__init__(app, conf)
self.memcache = FakeMemcache()
def __call__(self, env, start_response):
env['swift.cache'] = self.memcache
return self.app(env, start_response)
def fake_memcache_filter_factory(conf):
def filter_app(app):
return FakeMemcacheMiddleware(app, conf)
return filter_app
# swift.conf contents for in-process functional test runs
functests_swift_conf = '''
@ -133,6 +126,16 @@ max_file_size = %d
def in_process_setup(the_object_server=object_server):
print >>sys.stderr, 'IN-PROCESS SERVERS IN USE FOR FUNCTIONAL TESTS'
print >>sys.stderr, 'Using object_server: %s' % the_object_server.__name__
_dir = os.path.normpath(os.path.join(os.path.abspath(__file__),
os.pardir, os.pardir, os.pardir))
proxy_conf = os.path.join(_dir, 'etc', 'proxy-server.conf-sample')
if os.path.exists(proxy_conf):
print >>sys.stderr, 'Using proxy-server config from %s' % proxy_conf
else:
print >>sys.stderr, 'Failed to find conf file %s' % proxy_conf
return
monkey_patch_mimetools()
@ -159,7 +162,9 @@ def in_process_setup(the_object_server=object_server):
if constraints.SWIFT_CONSTRAINTS_LOADED:
# Use the swift constraints that are loaded for the test framework
# configuration
config.update(constraints.EFFECTIVE_CONSTRAINTS)
_c = dict((k, str(v))
for k, v in constraints.EFFECTIVE_CONSTRAINTS.items())
config.update(_c)
else:
# In-process swift constraints were not loaded, somethings wrong
raise SkipTest
@ -180,12 +185,9 @@ def in_process_setup(the_object_server=object_server):
'devices': _testdir,
'swift_dir': _testdir,
'mount_check': 'false',
'client_timeout': 4,
'client_timeout': '4',
'allow_account_management': 'true',
'account_autocreate': 'true',
'allowed_headers':
'content-disposition, content-encoding, x-delete-at,'
' x-object-manifest, x-static-large-object',
'allow_versions': 'True',
# Below are values used by the functional test framework, as well as
# by the various in-process swift servers
@ -257,7 +259,6 @@ def in_process_setup(the_object_server=object_server):
# Default to only 4 seconds for in-process functional test runs
eventlet.wsgi.WRITE_TIMEOUT = 4
prosrv = proxy_server.Application(config, logger=debug_logger('proxy'))
acc1srv = account_server.AccountController(
config, logger=debug_logger('acct1'))
acc2srv = account_server.AccountController(
@ -270,35 +271,16 @@ def in_process_setup(the_object_server=object_server):
config, logger=debug_logger('obj1'))
obj2srv = the_object_server.ObjectController(
config, logger=debug_logger('obj2'))
global _test_servers
_test_servers = \
(prosrv, acc1srv, acc2srv, con1srv, con2srv, obj1srv, obj2srv)
pipeline = [
catch_errors.filter_factory,
gatekeeper.filter_factory,
healthcheck.filter_factory,
proxy_logging.filter_factory,
fake_memcache_filter_factory,
container_sync.filter_factory,
bulk.filter_factory,
tempurl.filter_factory,
slo.filter_factory,
dlo.filter_factory,
ratelimit.filter_factory,
tempauth.filter_factory,
container_quotas.filter_factory,
account_quotas.filter_factory,
proxy_logging.filter_factory,
]
app = prosrv
import mock
for filter_factory in reversed(pipeline):
app_filter = filter_factory(config)
with mock.patch('swift.common.utils') as mock_utils:
mock_utils.get_logger.return_value = None
app = app_filter(app)
app.logger = prosrv.logger
logger = debug_logger('proxy')
def get_logger(name, *args, **kwargs):
return logger
with mock.patch('swift.common.utils.get_logger', get_logger):
with mock.patch('swift.common.middleware.memcache.MemcacheMiddleware',
FakeMemcacheMiddleware):
app = loadapp(proxy_conf, global_conf=config)
nl = utils.NullLogger()
prospa = eventlet.spawn(eventlet.wsgi.server, prolis, app, nl)
@ -315,7 +297,8 @@ def in_process_setup(the_object_server=object_server):
# Create accounts "test" and "test2"
def create_account(act):
ts = utils.normalize_timestamp(time())
partition, nodes = prosrv.account_ring.get_nodes(act)
account_ring = Ring(_testdir, ring_name='account')
partition, nodes = account_ring.get_nodes(act)
for node in nodes:
# Note: we are just using the http_connect method in the object
# controller here to talk to the account server nodes.
@ -348,7 +331,13 @@ def get_cluster_info():
# test.conf data
pass
else:
eff_constraints.update(cluster_info.get('swift', {}))
try:
eff_constraints.update(cluster_info['swift'])
except KeyError:
# Most likely the swift cluster has "expose_info = false" set
# in its proxy-server.conf file, so we'll just do the best we
# can.
print >>sys.stderr, "** Swift Cluster not exposing /info **"
# Finally, we'll allow any constraint present in the swift-constraints
# section of test.conf to override everything. Note that only those
@ -402,7 +391,10 @@ def setup_package():
config.update(get_config('func_test'))
if in_process:
in_process_setup()
in_mem_obj_env = os.environ.get('SWIFT_TEST_IN_MEMORY_OBJ')
in_mem_obj = utils.config_true_value(in_mem_obj_env)
in_process_setup(the_object_server=(
mem_object_server if in_mem_obj else object_server))
global web_front_end
web_front_end = config.get('web_front_end', 'integral')
@ -422,6 +414,7 @@ def setup_package():
global swift_test_key
global swift_test_tenant
global swift_test_perm
global swift_test_domain
if config:
swift_test_auth_version = str(config.get('auth_version', '1'))
@ -478,8 +471,13 @@ def setup_package():
swift_test_user[2] = config['username3']
swift_test_tenant[2] = config['account']
swift_test_key[2] = config['password3']
if 'username4' in config:
swift_test_user[3] = config['username4']
swift_test_tenant[3] = config['account4']
swift_test_key[3] = config['password4']
swift_test_domain[3] = config['domain4']
for _ in range(3):
for _ in range(4):
swift_test_perm[_] = swift_test_tenant[_] + ':' \
+ swift_test_user[_]
@ -501,6 +499,15 @@ def setup_package():
print >>sys.stderr, \
'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
global skip_if_not_v3
skip_if_not_v3 = (swift_test_auth_version != '3'
or not all([not skip,
swift_test_user[3],
swift_test_key[3]]))
if not skip and skip_if_not_v3:
print >>sys.stderr, \
'SKIPPING FUNCTIONAL TESTS SPECIFIC TO AUTH VERSION 3'
get_cluster_info()
@ -539,10 +546,10 @@ class InternalServerError(Exception):
pass
url = [None, None, None]
token = [None, None, None]
parsed = [None, None, None]
conn = [None, None, None]
url = [None, None, None, None]
token = [None, None, None, None]
parsed = [None, None, None, None]
conn = [None, None, None, None]
def connection(url):
@ -569,7 +576,8 @@ def retry(func, *args, **kwargs):
# access our own account by default
url_account = kwargs.pop('url_account', use_account + 1) - 1
os_options = {'user_domain_name': swift_test_domain[use_account],
'project_domain_name': swift_test_domain[use_account]}
while attempts <= retries:
attempts += 1
try:
@ -580,7 +588,7 @@ def retry(func, *args, **kwargs):
snet=False,
tenant_name=swift_test_tenant[use_account],
auth_version=swift_test_auth_version,
os_options={})
os_options=os_options)
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \

View File

@ -174,8 +174,10 @@ class Connection(object):
# unicode and this would cause troubles when doing
# no_safe_quote query.
self.storage_url = str('/%s/%s' % (x[3], x[4]))
self.account_name = str(x[4])
self.auth_user = auth_user
self.storage_token = storage_token
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_url, self.storage_token
@ -664,6 +666,32 @@ class File(Base):
return self.conn.make_request('COPY', self.path, hdrs=headers,
parms=parms) == 201
def copy_account(self, dest_account, dest_cont, dest_file,
hdrs=None, parms=None, cfg=None):
if hdrs is None:
hdrs = {}
if parms is None:
parms = {}
if cfg is None:
cfg = {}
if 'destination' in cfg:
headers = {'Destination': cfg['destination']}
elif cfg.get('no_destination'):
headers = {}
else:
headers = {'Destination-Account': dest_account,
'Destination': '%s/%s' % (dest_cont, dest_file)}
headers.update(hdrs)
if 'Destination-Account' in headers:
headers['Destination-Account'] = \
urllib.quote(headers['Destination-Account'])
if 'Destination' in headers:
headers['Destination'] = urllib.quote(headers['Destination'])
return self.conn.make_request('COPY', self.path, hdrs=headers,
parms=parms) == 201
def delete(self, hdrs=None, parms=None):
if hdrs is None:
hdrs = {}

View File

@ -777,6 +777,21 @@ class TestAccount(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata2(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
# TODO: Find the test that adds these and remove them.
headers = {'x-remove-account-meta-temp-url-key': 'remove',
'x-remove-account-meta-temp-url-key-2': 'remove'}
resp = retry(post, headers)
headers = {}
for x in xrange(self.max_meta_count):
headers['X-Account-Meta-%d' % x] = 'v'
@ -790,6 +805,16 @@ class TestAccount(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_bad_metadata3(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
headers = {}
header_value = 'k' * self.max_meta_value_length
size = 0
@ -812,5 +837,33 @@ class TestAccount(unittest.TestCase):
self.assertEqual(resp.status, 400)
class TestAccountInNonDefaultDomain(unittest.TestCase):
def setUp(self):
if tf.skip or tf.skip2 or tf.skip_if_not_v3:
raise SkipTest('AUTH VERSION 3 SPECIFIC TEST')
def test_project_domain_id_header(self):
# make sure account exists (assumes account auto create)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(post, use_account=4)
resp.read()
self.assertEqual(resp.status, 204)
# account in non-default domain should have a project domain id
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(head, use_account=4)
resp.read()
self.assertEqual(resp.status, 204)
self.assertTrue('X-Account-Project-Domain-Id' in resp.headers)
if __name__ == '__main__':
unittest.main()

View File

@ -404,6 +404,16 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata2(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path + '/' + self.name, '', headers)
return check_response(conn)
headers = {}
for x in xrange(self.max_meta_count):
headers['X-Container-Meta-%d' % x] = 'v'
@ -417,6 +427,16 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 400)
def test_POST_bad_metadata3(self):
if tf.skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path + '/' + self.name, '', headers)
return check_response(conn)
headers = {}
header_value = 'k' * self.max_meta_value_length
size = 0
@ -1419,7 +1439,7 @@ class TestContainer(unittest.TestCase):
self.assertEquals(headers.get('x-storage-policy'),
policy['name'])
# and test recreate with-out specifiying Storage Policy
# and test recreate with-out specifying Storage Policy
resp = retry(put)
resp.read()
self.assertEqual(resp.status, 202)
@ -1514,5 +1534,179 @@ class TestContainer(unittest.TestCase):
policy['name'])
class BaseTestContainerACLs(unittest.TestCase):
# subclasses can change the account in which container
# is created/deleted by setUp/tearDown
account = 1
def _get_account(self, url, token, parsed, conn):
return parsed.path
def _get_tenant_id(self, url, token, parsed, conn):
account = parsed.path
return account.replace('/v1/AUTH_', '', 1)
def setUp(self):
if tf.skip or tf.skip2 or tf.skip_if_not_v3:
raise SkipTest('AUTH VERSION 3 SPECIFIC TEST')
self.name = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put, use_account=self.account)
resp.read()
self.assertEqual(resp.status, 201)
def tearDown(self):
if tf.skip or tf.skip2 or tf.skip_if_not_v3:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name + '?format=json',
'', {'X-Auth-Token': token})
return check_response(conn)
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'/'.join([parsed.path, self.name, obj['name']]), '',
{'X-Auth-Token': token})
return check_response(conn)
while True:
resp = retry(get, use_account=self.account)
body = resp.read()
self.assert_(resp.status // 100 == 2, resp.status)
objs = json.loads(body)
if not objs:
break
for obj in objs:
resp = retry(delete, obj, use_account=self.account)
resp.read()
self.assertEqual(resp.status, 204)
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete, use_account=self.account)
resp.read()
self.assertEqual(resp.status, 204)
def _assert_cross_account_acl_granted(self, granted, grantee_account, acl):
'''
Check whether a given container ACL is granted when a user specified
by account_b attempts to access a container.
'''
# Obtain the first account's string
first_account = retry(self._get_account, use_account=self.account)
# Ensure we can't access the container with the grantee account
def get2(url, token, parsed, conn):
conn.request('GET', first_account + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get2, use_account=grantee_account)
resp.read()
self.assertEqual(resp.status, 403)
def put2(url, token, parsed, conn):
conn.request('PUT', first_account + '/' + self.name + '/object',
'test object', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put2, use_account=grantee_account)
resp.read()
self.assertEqual(resp.status, 403)
# Post ACL to the container
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Read': acl,
'X-Container-Write': acl})
return check_response(conn)
resp = retry(post, use_account=self.account)
resp.read()
self.assertEqual(resp.status, 204)
# Check access to container from grantee account with ACL in place
resp = retry(get2, use_account=grantee_account)
resp.read()
expected = 204 if granted else 403
self.assertEqual(resp.status, expected)
resp = retry(put2, use_account=grantee_account)
resp.read()
expected = 201 if granted else 403
self.assertEqual(resp.status, expected)
# Make the container private again
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': '',
'X-Container-Write': ''})
return check_response(conn)
resp = retry(post, use_account=self.account)
resp.read()
self.assertEqual(resp.status, 204)
# Ensure we can't access the container with the grantee account again
resp = retry(get2, use_account=grantee_account)
resp.read()
self.assertEqual(resp.status, 403)
resp = retry(put2, use_account=grantee_account)
resp.read()
self.assertEqual(resp.status, 403)
class TestContainerACLsAccount1(BaseTestContainerACLs):
def test_cross_account_acl_names_with_user_in_non_default_domain(self):
# names in acls are disallowed when grantee is in a non-default domain
acl = '%s:%s' % (tf.swift_test_tenant[3], tf.swift_test_user[3])
self._assert_cross_account_acl_granted(False, 4, acl)
def test_cross_account_acl_ids_with_user_in_non_default_domain(self):
# ids are allowed in acls when grantee is in a non-default domain
tenant_id = retry(self._get_tenant_id, use_account=4)
acl = '%s:%s' % (tenant_id, '*')
self._assert_cross_account_acl_granted(True, 4, acl)
def test_cross_account_acl_names_in_default_domain(self):
# names are allowed in acls when grantee and project are in
# the default domain
acl = '%s:%s' % (tf.swift_test_tenant[1], tf.swift_test_user[1])
self._assert_cross_account_acl_granted(True, 2, acl)
def test_cross_account_acl_ids_in_default_domain(self):
# ids are allowed in acls when grantee and project are in
# the default domain
tenant_id = retry(self._get_tenant_id, use_account=2)
acl = '%s:%s' % (tenant_id, '*')
self._assert_cross_account_acl_granted(True, 2, acl)
class TestContainerACLsAccount4(BaseTestContainerACLs):
account = 4
def test_cross_account_acl_names_with_project_in_non_default_domain(self):
# names in acls are disallowed when project is in a non-default domain
acl = '%s:%s' % (tf.swift_test_tenant[0], tf.swift_test_user[0])
self._assert_cross_account_acl_granted(False, 1, acl)
def test_cross_account_acl_ids_with_project_in_non_default_domain(self):
# ids are allowed in acls when project is in a non-default domain
tenant_id = retry(self._get_tenant_id, use_account=1)
acl = '%s:%s' % (tenant_id, '*')
self._assert_cross_account_acl_granted(True, 1, acl)
if __name__ == '__main__':
unittest.main()

View File

@ -35,6 +35,7 @@ class TestObject(unittest.TestCase):
self.containers = []
self._create_container(self.container)
self._create_container(self.container, use_account=2)
self.obj = uuid4().hex
@ -47,7 +48,7 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 201)
def _create_container(self, name=None, headers=None):
def _create_container(self, name=None, headers=None, use_account=1):
if not name:
name = uuid4().hex
self.containers.append(name)
@ -58,7 +59,7 @@ class TestObject(unittest.TestCase):
conn.request('PUT', parsed.path + '/' + name, '',
new_headers)
return check_response(conn)
resp = retry(put, name)
resp = retry(put, name, use_account=use_account)
resp.read()
self.assertEqual(resp.status, 201)
return name
@ -133,6 +134,45 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEquals(resp.status, 400)
def test_non_integer_x_delete_after(self):
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
'non_integer_x_delete_after'),
'', {'X-Auth-Token': token,
'Content-Length': '0',
'X-Delete-After': '*'})
return check_response(conn)
resp = retry(put)
body = resp.read()
self.assertEquals(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-After')
def test_non_integer_x_delete_at(self):
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
'non_integer_x_delete_at'),
'', {'X-Auth-Token': token,
'Content-Length': '0',
'X-Delete-At': '*'})
return check_response(conn)
resp = retry(put)
body = resp.read()
self.assertEquals(resp.status, 400)
self.assertEqual(body, 'Non-integer X-Delete-At')
def test_x_delete_at_in_the_past(self):
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
'x_delete_at_in_the_past'),
'', {'X-Auth-Token': token,
'Content-Length': '0',
'X-Delete-At': '0'})
return check_response(conn)
resp = retry(put)
body = resp.read()
self.assertEquals(resp.status, 400)
self.assertEqual(body, 'X-Delete-At in past')
def test_copy_object(self):
if tf.skip:
raise SkipTest
@ -207,6 +247,116 @@ class TestObject(unittest.TestCase):
resp.read()
self.assertEqual(resp.status, 204)
def test_copy_between_accounts(self):
if tf.skip:
raise SkipTest
source = '%s/%s' % (self.container, self.obj)
dest = '%s/%s' % (self.container, 'test_copy')
# get contents of source
def get_source(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, source),
'', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get_source)
source_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(source_contents, 'test')
acct = tf.parsed[0].path.split('/', 2)[2]
# copy source to dest with X-Copy-From-Account
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s' % (parsed.path, dest), '',
{'X-Auth-Token': token,
'Content-Length': '0',
'X-Copy-From-Account': acct,
'X-Copy-From': source})
return check_response(conn)
# try to put, will not succeed
# user does not have permissions to read from source
resp = retry(put, use_account=2)
self.assertEqual(resp.status, 403)
# add acl to allow reading from source
def post(url, token, parsed, conn):
conn.request('POST', '%s/%s' % (parsed.path, self.container), '',
{'X-Auth-Token': token,
'X-Container-Read': tf.swift_test_perm[1]})
return check_response(conn)
resp = retry(post)
self.assertEqual(resp.status, 204)
# retry previous put, now should succeed
resp = retry(put, use_account=2)
self.assertEqual(resp.status, 201)
# contents of dest should be the same as source
def get_dest(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, dest),
'', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get_dest, use_account=2)
dest_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(dest_contents, source_contents)
# delete the copy
def delete(url, token, parsed, conn):
conn.request('DELETE', '%s/%s' % (parsed.path, dest), '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete, use_account=2)
resp.read()
self.assertEqual(resp.status, 204)
# verify dest does not exist
resp = retry(get_dest, use_account=2)
resp.read()
self.assertEqual(resp.status, 404)
acct_dest = tf.parsed[1].path.split('/', 2)[2]
# copy source to dest with COPY
def copy(url, token, parsed, conn):
conn.request('COPY', '%s/%s' % (parsed.path, source), '',
{'X-Auth-Token': token,
'Destination-Account': acct_dest,
'Destination': dest})
return check_response(conn)
# try to copy, will not succeed
# user does not have permissions to write to destination
resp = retry(copy)
resp.read()
self.assertEqual(resp.status, 403)
# add acl to allow write to destination
def post(url, token, parsed, conn):
conn.request('POST', '%s/%s' % (parsed.path, self.container), '',
{'X-Auth-Token': token,
'X-Container-Write': tf.swift_test_perm[0]})
return check_response(conn)
resp = retry(post, use_account=2)
self.assertEqual(resp.status, 204)
# now copy will succeed
resp = retry(copy)
resp.read()
self.assertEqual(resp.status, 201)
# contents of dest should be the same as source
resp = retry(get_dest, use_account=2)
dest_contents = resp.read()
self.assertEqual(resp.status, 200)
self.assertEqual(dest_contents, source_contents)
# delete the copy
resp = retry(delete, use_account=2)
resp.read()
self.assertEqual(resp.status, 204)
def test_public_object(self):
if tf.skip:
raise SkipTest

View File

@ -25,6 +25,7 @@ import time
import unittest
import urllib
import uuid
from copy import deepcopy
import eventlet
from nose import SkipTest
@ -269,6 +270,8 @@ class TestAccount(Base):
containers)
def testQuotedWWWAuthenticateHeader(self):
# check that the www-authenticate header value with the swift realm
# is correctly quoted.
conn = Connection(tf.config)
conn.authenticate()
inserted_html = '<b>Hello World'
@ -277,9 +280,16 @@ class TestAccount(Base):
quoted_hax = urllib.quote(hax)
conn.connection.request('GET', '/v1/' + quoted_hax, None, {})
resp = conn.connection.getresponse()
resp_headers = resp.getheaders()
expected = ('www-authenticate', 'Swift realm="%s"' % quoted_hax)
self.assert_(expected in resp_headers)
resp_headers = dict(resp.getheaders())
self.assertTrue('www-authenticate' in resp_headers,
'www-authenticate not found in %s' % resp_headers)
actual = resp_headers['www-authenticate']
expected = 'Swift realm="%s"' % quoted_hax
# other middleware e.g. auth_token may also set www-authenticate
# headers in which case actual values will be a comma separated list.
# check that expected value is among the actual values
self.assertTrue(expected in actual,
'%s not found in %s' % (expected, actual))
class TestAccountUTF8(Base2, TestAccount):
@ -794,9 +804,22 @@ class TestFileEnv(object):
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
# creating another account and connection
# for account to account copy tests
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.account2 = cls.conn2.get_account()
cls.account2.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
@ -850,6 +873,62 @@ class TestFile(Base):
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
def testCopyAccount(self):
# makes sure to test encoded characters
source_filename = 'dealde%2Fl04 011e%204c8df/flash.png'
file_item = self.env.container.file(source_filename)
metadata = {Utils.create_ascii_name(): Utils.create_name()}
data = file_item.write_random()
file_item.sync_metadata(metadata)
dest_cont = self.env.account.container(Utils.create_name())
self.assert_(dest_cont.create())
acct = self.env.conn.account_name
# copy both from within and across containers
for cont in (self.env.container, dest_cont):
# copy both with and without initial slash
for prefix in ('', '/'):
dest_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
file_item.copy_account(acct,
'%s%s' % (prefix, cont),
dest_filename)
self.assert_(dest_filename in cont.files())
file_item = cont.file(dest_filename)
self.assert_(data == file_item.read())
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
dest_cont = self.env.account2.container(Utils.create_name())
self.assert_(dest_cont.create(hdrs={
'X-Container-Write': self.env.conn.user_acl
}))
acct = self.env.conn2.account_name
# copy both with and without initial slash
for prefix in ('', '/'):
dest_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
file_item.copy_account(acct,
'%s%s' % (prefix, dest_cont),
dest_filename)
self.assert_(dest_filename in dest_cont.files())
file_item = dest_cont.file(dest_filename)
self.assert_(data == file_item.read())
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
def testCopy404s(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@ -888,6 +967,77 @@ class TestFile(Base):
'%s%s' % (prefix, Utils.create_name()),
Utils.create_name()))
def testCopyAccount404s(self):
acct = self.env.conn.account_name
acct2 = self.env.conn2.account_name
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
file_item.write_random()
dest_cont = self.env.account.container(Utils.create_name())
self.assert_(dest_cont.create(hdrs={
'X-Container-Read': self.env.conn2.user_acl
}))
dest_cont2 = self.env.account2.container(Utils.create_name())
self.assert_(dest_cont2.create(hdrs={
'X-Container-Write': self.env.conn.user_acl,
'X-Container-Read': self.env.conn.user_acl
}))
for acct, cont in ((acct, dest_cont), (acct2, dest_cont2)):
for prefix in ('', '/'):
# invalid source container
source_cont = self.env.account.container(Utils.create_name())
file_item = source_cont.file(source_filename)
self.assert_(not file_item.copy_account(
acct,
'%s%s' % (prefix, self.env.container),
Utils.create_name()))
if acct == acct2:
# there is no such source container
# and foreign user can have no permission to read it
self.assert_status(403)
else:
self.assert_status(404)
self.assert_(not file_item.copy_account(
acct,
'%s%s' % (prefix, cont),
Utils.create_name()))
self.assert_status(404)
# invalid source object
file_item = self.env.container.file(Utils.create_name())
self.assert_(not file_item.copy_account(
acct,
'%s%s' % (prefix, self.env.container),
Utils.create_name()))
if acct == acct2:
# there is no such object
# and foreign user can have no permission to read it
self.assert_status(403)
else:
self.assert_status(404)
self.assert_(not file_item.copy_account(
acct,
'%s%s' % (prefix, cont),
Utils.create_name()))
self.assert_status(404)
# invalid destination container
file_item = self.env.container.file(source_filename)
self.assert_(not file_item.copy_account(
acct,
'%s%s' % (prefix, Utils.create_name()),
Utils.create_name()))
if acct == acct2:
# there is no such destination container
# and foreign user can have no permission to write there
self.assert_status(403)
else:
self.assert_status(404)
def testCopyNoDestinationHeader(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@ -942,6 +1092,49 @@ class TestFile(Base):
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
def testCopyFromAccountHeader(self):
acct = self.env.conn.account_name
src_cont = self.env.account.container(Utils.create_name())
self.assert_(src_cont.create(hdrs={
'X-Container-Read': self.env.conn2.user_acl
}))
source_filename = Utils.create_name()
file_item = src_cont.file(source_filename)
metadata = {}
for i in range(1):
metadata[Utils.create_ascii_name()] = Utils.create_name()
file_item.metadata = metadata
data = file_item.write_random()
dest_cont = self.env.account.container(Utils.create_name())
self.assert_(dest_cont.create())
dest_cont2 = self.env.account2.container(Utils.create_name())
self.assert_(dest_cont2.create(hdrs={
'X-Container-Write': self.env.conn.user_acl
}))
for cont in (src_cont, dest_cont, dest_cont2):
# copy both with and without initial slash
for prefix in ('', '/'):
dest_filename = Utils.create_name()
file_item = cont.file(dest_filename)
file_item.write(hdrs={'X-Copy-From-Account': acct,
'X-Copy-From': '%s%s/%s' % (
prefix,
src_cont.name,
source_filename)})
self.assert_(dest_filename in cont.files())
file_item = cont.file(dest_filename)
self.assert_(data == file_item.read())
self.assert_(file_item.initialize())
self.assert_(metadata == file_item.metadata)
def testCopyFromHeader404s(self):
source_filename = Utils.create_name()
file_item = self.env.container.file(source_filename)
@ -973,6 +1166,52 @@ class TestFile(Base):
self.env.container.name, source_filename)})
self.assert_status(404)
def testCopyFromAccountHeader404s(self):
acct = self.env.conn2.account_name
src_cont = self.env.account2.container(Utils.create_name())
self.assert_(src_cont.create(hdrs={
'X-Container-Read': self.env.conn.user_acl
}))
source_filename = Utils.create_name()
file_item = src_cont.file(source_filename)
file_item.write_random()
dest_cont = self.env.account.container(Utils.create_name())
self.assert_(dest_cont.create())
for prefix in ('', '/'):
# invalid source container
file_item = dest_cont.file(Utils.create_name())
self.assertRaises(ResponseError, file_item.write,
hdrs={'X-Copy-From-Account': acct,
'X-Copy-From': '%s%s/%s' %
(prefix,
Utils.create_name(),
source_filename)})
# looks like cached responses leak "not found"
# to un-authorized users, not going to fix it now, but...
self.assert_status([403, 404])
# invalid source object
file_item = self.env.container.file(Utils.create_name())
self.assertRaises(ResponseError, file_item.write,
hdrs={'X-Copy-From-Account': acct,
'X-Copy-From': '%s%s/%s' %
(prefix,
src_cont,
Utils.create_name())})
self.assert_status(404)
# invalid destination container
dest_cont = self.env.account.container(Utils.create_name())
file_item = dest_cont.file(Utils.create_name())
self.assertRaises(ResponseError, file_item.write,
hdrs={'X-Copy-From-Account': acct,
'X-Copy-From': '%s%s/%s' %
(prefix,
src_cont,
source_filename)})
self.assert_status(404)
def testNameLimit(self):
raise SkipTest('SOF constraints middleware enforces constraints.')
@ -1196,6 +1435,16 @@ class TestFile(Base):
cfg={'no_content_length': True})
self.assert_status(400)
# no content-length
self.assertRaises(ResponseError, file_item.write_random, file_length,
cfg={'no_content_length': True})
self.assert_status(411)
self.assertRaises(ResponseError, file_item.write_random, file_length,
hdrs={'transfer-encoding': 'gzip,chunked'},
cfg={'no_content_length': True})
self.assert_status(501)
# bad request types
#for req in ('LICK', 'GETorHEAD_base', 'container_info',
# 'best_response'):
@ -1598,6 +1847,30 @@ class TestDlo(Base):
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
def test_copy_account(self):
# dlo use same account and same container only
acct = self.env.conn.account_name
# Adding a new segment, copying the manifest, and then deleting the
# segment proves that the new object is really the concatenated
# segments and not just a manifest.
f_segment = self.env.container.file("%s/seg_lowerf" %
(self.env.segment_prefix))
f_segment.write('ffffffffff')
try:
man1_item = self.env.container.file('man1')
man1_item.copy_account(acct,
self.env.container.name,
"copied-man1")
finally:
# try not to leave this around for other tests to stumble over
f_segment.delete()
file_item = self.env.container.file('copied-man1')
file_contents = file_item.read()
self.assertEqual(
file_contents,
"aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffff")
def test_copy_manifest(self):
# Copying the manifest should result in another manifest
try:
@ -1794,6 +2067,14 @@ class TestSloEnv(object):
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
config2 = deepcopy(tf.config)
config2['account'] = tf.config['account2']
config2['username'] = tf.config['username2']
config2['password'] = tf.config['password2']
cls.conn2 = Connection(config2)
cls.conn2.authenticate()
cls.account2 = cls.conn2.get_account()
cls.account2.delete_containers()
if cls.slo_enabled is None:
cls.slo_enabled = 'slo' in cluster_info
@ -1976,6 +2257,29 @@ class TestSlo(Base):
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
def test_slo_copy_account(self):
acct = self.env.conn.account_name
# same account copy
file_item = self.env.container.file("manifest-abcde")
file_item.copy_account(acct, self.env.container.name, "copied-abcde")
copied = self.env.container.file("copied-abcde")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
# copy to different account
acct = self.env.conn2.account_name
dest_cont = self.env.account2.container(Utils.create_name())
self.assert_(dest_cont.create(hdrs={
'X-Container-Write': self.env.conn.user_acl
}))
file_item = self.env.container.file("manifest-abcde")
file_item.copy_account(acct, dest_cont, "copied-abcde")
copied = dest_cont.file("copied-abcde")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
def test_slo_copy_the_manifest(self):
file_item = self.env.container.file("manifest-abcde")
file_item.copy(self.env.container.name, "copied-abcde-manifest-only",
@ -1988,6 +2292,40 @@ class TestSlo(Base):
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
def test_slo_copy_the_manifest_account(self):
acct = self.env.conn.account_name
# same account
file_item = self.env.container.file("manifest-abcde")
file_item.copy_account(acct,
self.env.container.name,
"copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
copied = self.env.container.file("copied-abcde-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
try:
json.loads(copied_contents)
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
# different account
acct = self.env.conn2.account_name
dest_cont = self.env.account2.container(Utils.create_name())
self.assert_(dest_cont.create(hdrs={
'X-Container-Write': self.env.conn.user_acl
}))
file_item.copy_account(acct,
dest_cont,
"copied-abcde-manifest-only",
parms={'multipart-manifest': 'get'})
copied = dest_cont.file("copied-abcde-manifest-only")
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
try:
json.loads(copied_contents)
except ValueError:
self.fail("COPY didn't copy the manifest (invalid json on GET)")
def test_slo_get_the_manifest(self):
manifest = self.env.container.file("manifest-abcde")
got_body = manifest.read(parms={'multipart-manifest': 'get'})

View File

@ -1,30 +0,0 @@
# Copyright (c) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tests for swiftonfile.swift.obj.server subclass """
import unittest
from nose import SkipTest
import swiftonfile.swift.obj.server as server
class TestObjServer(unittest.TestCase):
"""
Tests for object server subclass.
"""
def test_constructor(self):
raise SkipTest

View File

@ -21,7 +21,7 @@ deps =
# Note: pip supports installing from git repos.
# https://pip.pypa.io/en/latest/reference/pip_install.html#git
# Example: git+https://github.com/openstack/swift.git@2.0.0
https://launchpad.net/swift/juno/2.1.0/+download/swift-2.1.0.tar.gz
https://launchpad.net/swift/kilo/2.2.1/+download/swift-2.2.1.tar.gz
-r{toxinidir}/test-requirements.txt
changedir = {toxinidir}/test/unit
commands = nosetests -v {posargs}