x/swiftonhpss
Code Issues Proposed changes

Rebase to OpenStack Swift Havana (1.10.0)

Browse Source 
Change-Id: I90821230a1a7100c74d97cccc9c445251d0f65e7
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/6157
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Luis Pabon <lpabon@redhat.com>
This commit is contained in:
Peter Portante 2013-10-24 16:15:25 -04:00 committed by Luis Pabon
parent 6b8d7c5919
commit 286a1308db
24 changed files with 2130 additions and 1466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gluster/swift/__init__.py
View File

@ -44,6 +44,6 @@ class PkgInfo(object):
###
### Change the Package version here
###
_pkginfo = PkgInfo('1.9.1', '0', 'glusterfs-openstack-swift', False)
_pkginfo = PkgInfo('1.10.0', '0', 'glusterfs-openstack-swift', False)
__version__ = _pkginfo.pretty_version
__canonical_version__ = _pkginfo.canonical_version

10
gluster/swift/common/fs_utils.py
View File

@ -32,7 +32,7 @@ class Fake_file(object):
return 0
def read(self, count):
return 0
return None
def fileno(self):
return -1
@ -265,6 +265,14 @@ def do_fsync(fd):
err.errno, '%s, os.fsync("%s")' % (err.strerror, fd))
def do_fdatasync(fd):
try:
os.fdatasync(fd)
except OSError as err:
raise GlusterFileSystemOSError(
err.errno, '%s, os.fdatasync("%s")' % (err.strerror, fd))
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path

46
gluster/swift/common/utils.py
View File

@ -23,8 +23,9 @@ from hashlib import md5
from eventlet import sleep
import cPickle as pickle
from swift.common.utils import normalize_timestamp
from gluster.swift.common.exceptions import GlusterFileSystemIOError
from gluster.swift.common.fs_utils import do_rename, do_fsync, os_path, \
do_stat, do_listdir, do_walk, do_rmdir
do_stat, do_fstat, do_listdir, do_walk, do_rmdir
from gluster.swift.common import Glusterfs
X_CONTENT_TYPE = 'Content-Type'
@ -55,18 +56,6 @@ PICKLE_PROTOCOL = 2
CHUNK_SIZE = 65536
class GlusterFileSystemOSError(OSError):
# Having our own class means the name will show up in the stack traces
# recorded in the log files.
pass
class GlusterFileSystemIOError(IOError):
# Having our own class means the name will show up in the stack traces
# recorded in the log files.
pass
def read_metadata(path_or_fd):
"""
Helper function to read the pickled metadata from a File/Directory.
@ -320,16 +309,8 @@ def get_account_details(acc_path):
return container_list, container_count
def _get_etag(path):
"""
FIXME: It would be great to have a translator that returns the md5sum() of
the file as an xattr that can be simply fetched.
Since we don't have that we should yield after each chunk read and
computed so that we don't consume the worker thread.
"""
def _read_for_etag(fp):
etag = md5()
with open(path, 'rb') as fp:
while True:
chunk = fp.read(CHUNK_SIZE)
if chunk:
@ -345,10 +326,31 @@ def _get_etag(path):
return etag.hexdigest()
def _get_etag(path):
"""
FIXME: It would be great to have a translator that returns the md5sum() of
the file as an xattr that can be simply fetched.
Since we don't have that we should yield after each chunk read and
computed so that we don't consume the worker thread.
"""
if isinstance(path, int):
with os.fdopen(os.dup(path), 'rb') as fp:
etag = _read_for_etag(fp)
os.lseek(path, 0, os.SEEK_SET)
else:
with open(path, 'rb') as fp:
etag = _read_for_etag(fp)
return etag
def get_object_metadata(obj_path):
"""
Return metadata of object.
"""
if isinstance(obj_path, int):
stats = do_fstat(obj_path)
else:
stats = do_stat(obj_path)
if not stats:
metadata = {}

278
gluster/swift/obj/diskfile.py
View File

@ -32,10 +32,12 @@ from swift.common.utils import TRUE_VALUES, drop_buffer_cache, ThreadPool
from swift.common.exceptions import DiskFileNotExist, DiskFileError, \
DiskFileNoSpace, DiskFileDeviceUnavailable
from gluster.swift.common.exceptions import GlusterFileSystemOSError
from gluster.swift.common.exceptions import GlusterFileSystemOSError, \
GlusterFileSystemIOError
from gluster.swift.common.Glusterfs import mount
from gluster.swift.common.fs_utils import do_fstat, do_open, do_close, \
do_unlink, do_chown, os_path, do_fsync, do_fchown, do_stat, Fake_file
do_unlink, do_chown, os_path, do_fsync, do_fchown, do_stat, do_write, \
do_fdatasync, do_rename, Fake_file
from gluster.swift.common.utils import read_metadata, write_metadata, \
validate_object, create_object_metadata, rmobjdir, dir_is_object, \
get_object_metadata
@ -45,7 +47,6 @@ from gluster.swift.common.utils import X_CONTENT_LENGTH, X_CONTENT_TYPE, \
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.obj.diskfile import DiskFile as SwiftDiskFile
from swift.obj.diskfile import DiskWriter as SwiftDiskWriter
# FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will
# be back ported. See http://www.python.org/dev/peps/pep-0433/
@ -278,7 +279,7 @@ def _adjust_metadata(metadata):
return metadata
class DiskWriter(SwiftDiskWriter):
class DiskWriter(object):
"""
Encapsulation of the write context for servicing PUT REST API
requests. Serves as the context manager object for DiskFile's writer()
@ -286,46 +287,37 @@ class DiskWriter(SwiftDiskWriter):
We just override the put() method for Gluster.
"""
def put(self, metadata, extension='.data'):
def __init__(self, disk_file, fd, tmppath, threadpool):
self.disk_file = disk_file
self.fd = fd
self.tmppath = tmppath
self.upload_size = 0
self.last_sync = 0
self.threadpool = threadpool
def write(self, chunk):
"""
Finalize writing the file on disk, and renames it from the temp file
to the real location. This should be called after the data has been
written to the temp file.
Write a chunk of data into the temporary file.
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
:param chunk: the chunk of data to write as a string object
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
assert self.tmppath is not None
metadata = _adjust_metadata(metadata)
df = self.disk_file
def _write_entire_chunk(chunk):
while chunk:
written = do_write(self.fd, chunk)
self.upload_size += written
chunk = chunk[written:]
if dir_is_object(metadata):
if not df.data_file:
# Does not exist, create it
data_file = os.path.join(df._obj_path, df._obj)
_, df.metadata = self.threadpool.force_run_in_thread(
df._create_dir_object, data_file, metadata)
df.data_file = os.path.join(df._container_path, data_file)
elif not df.is_dir:
# Exists, but as a file
raise DiskFileError('DiskFile.put(): directory creation failed'
' since the target, %s, already exists as'
' a file' % df.data_file)
return
self.threadpool.run_in_thread(_write_entire_chunk, chunk)
if df._is_dir:
# A pre-existing directory already exists on the file
# system, perhaps gratuitously created when another
# object was created, or created externally to Swift
# REST API servicing (UFO use case).
raise DiskFileError('DiskFile.put(): file creation failed since'
' the target, %s, already exists as a'
' directory' % df.data_file)
# For large files sync every 512MB (by default) written
diff = self.upload_size - self.last_sync
if diff >= self.disk_file.bytes_per_sync:
self.threadpool.force_run_in_thread(do_fdatasync, self.fd)
drop_buffer_cache(self.fd, self.last_sync, diff)
self.last_sync = self.upload_size
def finalize_put():
def _finalize_put(self, metadata):
# Write out metadata before fsync() to ensure it is also forced to
# disk.
write_metadata(self.fd, metadata)
@ -335,6 +327,7 @@ class DiskWriter(SwiftDiskWriter):
# the pages (now that after fsync the pages will be all
# clean).
do_fsync(self.fd)
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
# (pages all clean).
@ -344,11 +337,12 @@ class DiskWriter(SwiftDiskWriter):
# exists, so we can just rename it directly without using Swift's
# swift.common.utils.renamer(), which makes the directory path and
# adds extra stat() calls.
df = self.disk_file
data_file = os.path.join(df.put_datadir, df._obj)
attempts = 1
while True:
try:
os.rename(self.tmppath, data_file)
do_rename(self.tmppath, data_file)
except OSError as err:
if err.errno in (errno.ENOENT, errno.EIO) \
and attempts < MAX_RENAME_ATTEMPTS:
@ -413,13 +407,54 @@ class DiskWriter(SwiftDiskWriter):
# in a thread.
do_close(self.fd)
self.threadpool.force_run_in_thread(finalize_put)
def put(self, metadata, extension='.data'):
"""
Finalize writing the file on disk, and renames it from the temp file
to the real location. This should be called after the data has been
written to the temp file.
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
assert self.tmppath is not None
metadata = _adjust_metadata(metadata)
df = self.disk_file
if dir_is_object(metadata):
if not df.data_file:
# Does not exist, create it
data_file = os.path.join(df._obj_path, df._obj)
_, df._metadata = self.threadpool.force_run_in_thread(
df._create_dir_object, data_file, metadata)
df.data_file = os.path.join(df._container_path, data_file)
elif not df._is_dir:
# Exists, but as a file
raise DiskFileError('DiskFile.put(): directory creation failed'
' since the target, %s, already exists as'
' a file' % df.data_file)
return
try:
self.threadpool.force_run_in_thread(self._finalize_put, metadata)
except GlusterFileSystemOSError as err:
if err.errno == errno.EISDIR:
# A pre-existing directory already exists on the file
# system, perhaps gratuitously created when another
# object was created, or created externally to Swift
# REST API servicing (UFO use case).
raise DiskFileError('DiskFile.put(): file creation failed'
' since the target, %s, already exists as'
' a directory' % df.data_file)
raise
# Avoid the unlink() system call as part of the mkstemp context
# cleanup
self.tmppath = None
df.metadata = metadata
df._metadata = metadata
df._filter_metadata()
# Mark that it actually exists now
@ -443,7 +478,6 @@ class DiskFile(SwiftDiskFile):
:param container: container name for the object
:param obj: object name for the object
:param logger: logger object for writing out log file messages
:param keep_data_fp: if True, don't close the fp, otherwise close it
:param disk_chunk_Size: size of chunks on file reads
:param bytes_per_sync: number of bytes between fdatasync calls
:param iter_hook: called when __iter__ returns a chunk
@ -456,18 +490,15 @@ class DiskFile(SwiftDiskFile):
"""
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
logger, disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
bytes_per_sync=DEFAULT_BYTES_PER_SYNC, iter_hook=None,
threadpool=None, obj_dir='objects', mount_check=False,
disallowed_metadata_keys=None, uid=DEFAULT_UID,
gid=DEFAULT_GID):
uid=DEFAULT_UID, gid=DEFAULT_GID):
if mount_check and not mount(path, device):
raise DiskFileDeviceUnavailable()
self.disk_chunk_size = disk_chunk_size
self.bytes_per_sync = bytes_per_sync
self.iter_hook = iter_hook
self.threadpool = threadpool or ThreadPool(nthreads=0)
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
@ -491,59 +522,78 @@ class DiskFile(SwiftDiskFile):
self.put_datadir = self.datadir
self._is_dir = False
self.logger = logger
self.metadata = {}
self.meta_file = None
self._metadata = None
# Don't store a value for data_file until we know it exists.
self.data_file = None
self._data_file_size = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.suppress_file_closing = False
self._verify_close = False
self.threadpool = threadpool or ThreadPool(nthreads=0)
# FIXME(portante): this attribute is set after open and affects the
# behavior of the class (i.e. public interface)
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
self.suppress_file_closing = False
# Don't store a value for data_file until we know it exists.
self.data_file = None
def open(self, verify_close=False):
"""
Open the file and read the metadata.
This method must populate the _metadata attribute.
:param verify_close: force implicit close to verify_file, no effect on
explicit close.
:raises DiskFileCollision: on md5 collision
"""
data_file = os.path.join(self.put_datadir, self._obj)
try:
stats = do_stat(data_file)
except OSError as err:
if err.errno == errno.ENOTDIR:
return
fd = do_open(data_file, os.O_RDONLY | os.O_EXCL)
except GlusterFileSystemOSError as err:
self.logger.exception(
"Error opening file, %s :: %s", data_file, err)
else:
try:
stats = do_fstat(fd)
except GlusterFileSystemOSError as err:
self.logger.exception(
"Error stat'ing open file, %s :: %s", data_file, err)
else:
if not stats:
return
self.data_file = data_file
self._is_dir = stat.S_ISDIR(stats.st_mode)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.data_file = data_file
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self._metadata = read_metadata(fd)
if not self._metadata:
create_object_metadata(fd)
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
self._filter_metadata()
if keep_data_fp:
if not self._is_dir:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
else:
if self._is_dir:
# Use a fake file handle to satisfy the super class's
# __iter__ method requirement when dealing with
# directories as objects.
os.close(fd)
self.fp = Fake_file(data_file)
else:
self.fp = os.fdopen(fd, 'rb')
self._verify_close = verify_close
self._metadata = self._metadata or {}
return self
def drop_cache(self, fd, offset, length):
def _drop_cache(self, fd, offset, length):
if fd >= 0:
super(DiskFile, self).drop_cache(fd, offset, length)
super(DiskFile, self)._drop_cache(fd, offset, length)
def close(self, verify_file=True):
"""
@ -555,12 +605,17 @@ class DiskFile(SwiftDiskFile):
if self.fp:
do_close(self.fp)
self.fp = None
self._metadata = None
self._data_file_size = None
self._verify_close = False
def _filter_metadata(self):
if X_TYPE in self.metadata:
self.metadata.pop(X_TYPE)
if X_OBJECT_TYPE in self.metadata:
self.metadata.pop(X_OBJECT_TYPE)
if self._metadata is None:
return
if X_TYPE in self._metadata:
self._metadata.pop(X_TYPE)
if X_OBJECT_TYPE in self._metadata:
self._metadata.pop(X_OBJECT_TYPE)
def _create_dir_object(self, dir_path, metadata=None):
"""
@ -619,7 +674,7 @@ class DiskFile(SwiftDiskFile):
return True, newmd
@contextmanager
def writer(self, size=None):
def create(self, size=None):
"""
Contextmanager to make a temporary file, optionally of a specified
initial size.
@ -721,27 +776,11 @@ class DiskFile(SwiftDiskFile):
if tombstone:
# We don't write tombstone files. So do nothing.
return
assert self.data_file is not None, \
"put_metadata: no file to put metadata into"
metadata = _adjust_metadata(metadata)
self.threadpool.run_in_thread(write_metadata, self.data_file, metadata)
self.metadata = metadata
self._filter_metadata()
data_file = os.path.join(self.put_datadir, self._obj)
self.threadpool.run_in_thread(write_metadata, data_file, metadata)
def unlinkold(self, timestamp):
"""
Remove any older versions of the object file. Any file that has an
older timestamp than timestamp will be deleted.
:param timestamp: timestamp to compare with each file
"""
if not self.metadata or self.metadata[X_TIMESTAMP] >= timestamp:
return
assert self.data_file, \
"Have metadata, %r, but no data_file" % self.metadata
def _unlinkold():
def _delete(self):
if self._is_dir:
# Marker, or object, directory.
#
@ -772,12 +811,33 @@ class DiskFile(SwiftDiskFile):
else:
dirname = os.path.dirname(dirname)
self.threadpool.run_in_thread(_unlinkold)
def delete(self, timestamp):
"""
Remove any older versions of the object file. Any file that has an
older timestamp than timestamp will be deleted.
self.metadata = {}
:param timestamp: timestamp to compare with each file
"""
timestamp_fl = float(timestamp)
data_file = os.path.join(self.put_datadir, self._obj)
try:
metadata = read_metadata(data_file)
except (GlusterFileSystemIOError, GlusterFileSystemOSError) as err:
if err.errno != errno.ENOENT:
raise
else:
try:
old_ts = float(metadata[X_TIMESTAMP]) >= timestamp_fl
except (KeyError, ValueError):
# If no X-Timestamp to compare against, or the timestamp is
# not a valid float, we'll just delete the object anyways.
old_ts = False
if not old_ts:
self.threadpool.run_in_thread(self._delete)
self._metadata = {}
self.data_file = None
def get_data_file_size(self):
def _get_data_file_size(self):
"""
Returns the os_path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata, or if
@ -795,12 +855,12 @@ class DiskFile(SwiftDiskFile):
if self.data_file:
def _old_getsize():
file_size = os_path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if X_CONTENT_LENGTH in self._metadata:
metadata_size = int(self._metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
# FIXME - bit rot detection?
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
self._metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self._metadata)
return file_size
file_size = self.threadpool.run_in_thread(_old_getsize)
return file_size

2
gluster/swift/obj/server.py
View File

@ -39,8 +39,6 @@ class ObjectController(server.ObjectController):
kwargs.setdefault('disk_chunk_size', self.disk_chunk_size)
kwargs.setdefault('threadpool', self.threadpools[device])
kwargs.setdefault('obj_dir', server.DATADIR)
kwargs.setdefault('disallowed_metadata_keys',
server.DISALLOWED_HEADERS)
return DiskFile(self.devices, device, partition, account,
container, obj, self.logger, **kwargs)

7
gluster/swift/proxy/server.py
View File

@ -18,11 +18,14 @@
# needs
import gluster.swift.common.constraints # noqa
from swift.proxy import server
from swift.proxy.server import Application, mimetypes # noqa
from swift.proxy.controllers import AccountController # noqa
from swift.proxy.controllers import ObjectController # noqa
from swift.proxy.controllers import ContainerController # noqa
def app_factory(global_conf, **local_conf): # noqa
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()
conf.update(local_conf)
return server.Application(conf)
return Application(conf)

10
glusterfs-openstack-swift.spec
View File

@ -39,11 +39,11 @@ BuildRequires: python-setuptools
Requires : memcached
Requires : openssl
Requires : python
Requires : openstack-swift = 1.9.1
Requires : openstack-swift-account = 1.9.1
Requires : openstack-swift-container = 1.9.1
Requires : openstack-swift-object = 1.9.1
Requires : openstack-swift-proxy = 1.9.1
Requires : openstack-swift = 1.10.0
Requires : openstack-swift-account = 1.10.0
Requires : openstack-swift-container = 1.10.0
Requires : openstack-swift-object = 1.10.0
Requires : openstack-swift-proxy = 1.10.0
Obsoletes: glusterfs-swift-plugin
Obsoletes: glusterfs-swift
Obsoletes: glusterfs-ufo

2
modules/swift

@ -1 +1 @@
Subproject commit 4bd9e4584d31eb37c7e30e555daeb6b90703ee3a
Subproject commit 7accddf1c3f54f67cf29d6eb69e416f798af6e23

36
test/__init__.py
View File

@ -1,14 +1,44 @@
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# See http://code.google.com/p/python-nose/issues/detail?id=373
# The code below enables nosetests to work with i18n _() blocks
import __builtin__
import sys
import os
try:
from unittest.util import safe_repr
except ImportError:
# Probably py26
_MAX_LENGTH = 80
def safe_repr(obj, short=False):
try:
result = repr(obj)
except Exception:
result = object.__repr__(obj)
if not short or len(result) < _MAX_LENGTH:
return result
return result[:_MAX_LENGTH] + ' [truncated]...'
# make unittests pass on all locale
import swift
setattr(swift, 'gettext_', lambda x: x)
from swift.common.utils import readconf
setattr(__builtin__, '_', lambda x: x)
# Work around what seems to be a Python bug.
# c.f. https://bugs.launchpad.net/swift/+bug/820185.

95
test/functional/swift_test_client.py
View File

@ -1,19 +1,4 @@
# Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2013 Red Hat, Inc.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -43,6 +28,8 @@ from nose import SkipTest
from xml.dom import minidom
from swiftclient import get_auth
from test import safe_repr
class AuthenticationFailed(Exception):
pass
@ -146,11 +133,9 @@ class Connection(object):
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(auth_url,
auth_user, self.password,
snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
(storage_url, storage_token) = get_auth(
auth_url, auth_user, self.password, snet=False,
tenant_name=self.account, auth_version=self.auth_version,
os_options={})
if not (storage_url and storage_token):
@ -233,18 +218,22 @@ class Connection(object):
self.response = None
try_count = 0
fail_messages = []
while try_count < 5:
try_count += 1
try:
self.response = try_request()
except httplib.HTTPException:
except httplib.HTTPException as e:
fail_messages.append(safe_repr(e))
continue
if self.response.status == 401:
fail_messages.append("Response 401")
self.authenticate()
continue
elif self.response.status == 503:
fail_messages.append("Response 503")
if try_count != 5:
time.sleep(5)
continue
@ -254,7 +243,11 @@ class Connection(object):
if self.response:
return self.response.status
raise RequestError('Unable to complete http request')
request = "{method} {path} headers: {headers} data: {data}".format(
method=method, path=path, headers=headers, data=data)
raise RequestError('Unable to complete http request: %s. '
'Attempts: %s, Failures: %s' %
(request, len(fail_messages), fail_messages))
def put_start(self, path, hdrs={}, parms={}, cfg={}, chunked=False):
self.http_connect()
@ -329,21 +322,21 @@ class Account(Base):
return Container(self.conn, self.name, container_name)
def containers(self, hdrs={}, parms={}, cfg={}):
format = parms.get('format', None)
if format not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format)
if format is None and 'format' in parms:
format_type = parms.get('format', None)
if format_type not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format_type)
if format_type is None and 'format' in parms:
del parms['format']
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
parms=parms, cfg=cfg)
if status == 200:
if format == 'json':
if format_type == 'json':
conts = json.loads(self.conn.response.read())
for cont in conts:
cont['name'] = cont['name'].encode('utf-8')
return conts
elif format == 'xml':
elif format_type == 'xml':
conts = []
tree = minidom.parseString(self.conn.response.read())
for x in tree.getElementsByTagName('container'):
@ -406,8 +399,8 @@ class Container(Base):
def delete_files(self):
for f in listing_items(self.files):
file = self.file(f)
if not file.delete():
file_item = self.file(f)
if not file_item.delete():
return False
return listing_empty(self.files)
@ -419,37 +412,39 @@ class Container(Base):
return File(self.conn, self.account, self.name, file_name)
def files(self, hdrs={}, parms={}, cfg={}):
format = parms.get('format', None)
if format not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format)
if format is None and 'format' in parms:
format_type = parms.get('format', None)
if format_type not in [None, 'json', 'xml']:
raise RequestError('Invalid format: %s' % format_type)
if format_type is None and 'format' in parms:
del parms['format']
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
parms=parms, cfg=cfg)
if status == 200:
if format == 'json':
if format_type == 'json':
files = json.loads(self.conn.response.read())
for file in files:
file['name'] = file['name'].encode('utf-8')
file['content_type'] = file['content_type'].encode('utf-8')
for file_item in files:
file_item['name'] = file_item['name'].encode('utf-8')
file_item['content_type'] = file_item['content_type'].\
encode('utf-8')
return files
elif format == 'xml':
elif format_type == 'xml':
files = []
tree = minidom.parseString(self.conn.response.read())
for x in tree.getElementsByTagName('object'):
file = {}
file_item = {}
for key in ['name', 'hash', 'bytes', 'content_type',
'last_modified']:
file[key] = x.getElementsByTagName(key)[0].\
file_item[key] = x.getElementsByTagName(key)[0].\
childNodes[0].nodeValue
files.append(file)
files.append(file_item)
for file in files:
file['name'] = file['name'].encode('utf-8')
file['content_type'] = file['content_type'].encode('utf-8')
for file_item in files:
file_item['name'] = file_item['name'].encode('utf-8')
file_item['content_type'] = file_item['content_type'].\
encode('utf-8')
return files
else:
content = self.conn.response.read()
@ -616,11 +611,11 @@ class File(Base):
callback=None, cfg={}):
if size > 0:
range = 'bytes=%d-%d' % (offset, (offset + size) - 1)
range_string = 'bytes=%d-%d' % (offset, (offset + size) - 1)
if hdrs:
hdrs['Range'] = range
hdrs['Range'] = range_string
else:
hdrs = {'Range': range}
hdrs = {'Range': range_string}
status = self.conn.make_request('GET', self.path, hdrs=hdrs,
cfg=cfg)
@ -736,7 +731,7 @@ class File(Base):
callback(transferred, self.size)
self.conn.put_end()
except socket.timeout, err:
except socket.timeout as err:
raise err
if (self.conn.response.status < 200) or \

652
test/functional/tests.py
View File

File diff suppressed because it is too large Load Diff

38
test/functionalnosetests/swift_testing.py
View File

@ -1,19 +1,4 @@
# Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2013 Red Hat, Inc.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -28,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from httplib import HTTPException
import os
import socket
import sys
@ -35,7 +21,7 @@ from time import sleep
from test import get_config
from swiftclient import get_auth, http_connection, HTTPException
from swiftclient import get_auth, http_connection
conf = get_config('func_test')
web_front_end = conf.get('web_front_end', 'integral')
@ -57,8 +43,8 @@ if conf:
if 'auth_prefix' not in conf:
conf['auth_prefix'] = '/'
try:
swift_test_auth += \
'://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf
suffix = '://%(auth_host)s:%(auth_port)s%(auth_prefix)s' % conf
swift_test_auth += suffix
except KeyError:
pass # skip
@ -71,17 +57,17 @@ if conf:
swift_test_user[0] = '%(username)s' % conf
swift_test_key[0] = conf['password']
try:
swift_test_user[1] = '%s%s' % \
('%s:' % conf['account2'] if 'account2' in conf else '',
swift_test_user[1] = '%s%s' % (
'%s:' % conf['account2'] if 'account2' in conf else '',
conf['username2'])
swift_test_key[1] = conf['password2']
except KeyError, err:
except KeyError as err:
pass # old conf, no second account tests can be run
try:
swift_test_user[2] = '%s%s' % ('%s:' % conf['account'] if 'account'
in conf else '', conf['username3'])
swift_test_key[2] = conf['password3']
except KeyError, err:
except KeyError as err:
pass # old conf, no third account tests can be run
for _ in range(3):
@ -99,7 +85,8 @@ if conf:
swift_test_key[2] = conf['password3']
for _ in range(3):
swift_test_perm[_] = swift_test_tenant[_] + ':' + swift_test_user[_]
swift_test_perm[_] = swift_test_tenant[_] + ':' \
+ swift_test_user[_]
skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
if skip:
@ -160,7 +147,8 @@ def retry(func, *args, **kwargs):
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account], *args, **kwargs)
parsed[use_account], conn[use_account],
*args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise

33
test/functionalnosetests/test_account.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/python
# Copyright (c) 2010-2013 OpenStack, LLC.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,22 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from nose import SkipTest
@ -45,16 +29,20 @@ class TestAccount(unittest.TestCase):
def test_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, value):
conn.request('POST', parsed.path, '',
{'X-Auth-Token': token, 'X-Account-Meta-Test': value})
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
def get(url, token, parsed, conn):
conn.request('GET', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(post, '')
resp.read()
self.assertEquals(resp.status, 204)
@ -121,13 +109,16 @@ class TestAccount(unittest.TestCase):
def test_multi_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
conn.request('POST', parsed.path, '',
{'X-Auth-Token': token, name: value})
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(post, 'X-Account-Meta-One', '1')
resp.read()
self.assertEquals(resp.status, 204)
@ -147,16 +138,19 @@ class TestAccount(unittest.TestCase):
def test_bad_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path, '', headers)
return check_response(conn)
resp = retry(post,
{'X-Account-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(post,
resp = retry(
post,
{'X-Account-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
resp.read()
self.assertEquals(resp.status, 400)
@ -165,7 +159,8 @@ class TestAccount(unittest.TestCase):
{'X-Account-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(post,
resp = retry(
post,
{'X-Account-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
resp.read()
self.assertEquals(resp.status, 400)

121
test/functionalnosetests/test_container.py
View File

@ -1,21 +1,6 @@
#!/usr/bin/python
# Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2013 Red Hat, Inc.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -48,10 +33,12 @@ class TestContainer(unittest.TestCase):
if skip:
raise SkipTest
self.name = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEquals(resp.status, 201)
@ -59,15 +46,18 @@ class TestContainer(unittest.TestCase):
def tearDown(self):
if skip:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name + '?format=json',
'', {'X-Auth-Token': token})
return check_response(conn)
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'/'.join([parsed.path, self.name, obj['name']]), '',
{'X-Auth-Token': token})
return check_response(conn)
while True:
resp = retry(get)
body = resp.read()
@ -79,10 +69,12 @@ class TestContainer(unittest.TestCase):
resp = retry(delete, obj)
resp.read()
self.assertEquals(resp.status, 204)
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
@ -90,14 +82,17 @@ class TestContainer(unittest.TestCase):
def test_multi_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, name, value):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, name: value})
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(post, 'X-Container-Meta-One', '1')
resp.read()
self.assertEquals(resp.status, 204)
@ -159,22 +154,28 @@ class TestContainer(unittest.TestCase):
def test_PUT_metadata(self):
if skip:
raise SkipTest
def put(url, token, parsed, conn, name, value):
conn.request('PUT', parsed.path + '/' + name, '',
{'X-Auth-Token': token, 'X-Container-Meta-Test': value})
{'X-Auth-Token': token,
'X-Container-Meta-Test': value})
return check_response(conn)
def head(url, token, parsed, conn, name):
conn.request('HEAD', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
def get(url, token, parsed, conn, name):
conn.request('GET', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
def delete(url, token, parsed, conn, name):
conn.request('DELETE', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
name = uuid4().hex
resp = retry(put, name, 'Value')
resp.read()
@ -210,18 +211,23 @@ class TestContainer(unittest.TestCase):
def test_POST_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, value):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Meta-Test': value})
{'X-Auth-Token': token,
'X-Container-Meta-Test': value})
return check_response(conn)
def head(url, token, parsed, conn):
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(head)
resp.read()
self.assert_(resp.status in (200, 204), resp.status)
@ -245,17 +251,21 @@ class TestContainer(unittest.TestCase):
def test_PUT_bad_metadata(self):
if skip:
raise SkipTest
def put(url, token, parsed, conn, name, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('PUT', parsed.path + '/' + name, '', headers)
return check_response(conn)
def delete(url, token, parsed, conn, name):
conn.request('DELETE', parsed.path + '/' + name, '',
{'X-Auth-Token': token})
return check_response(conn)
name = uuid4().hex
resp = retry(put, name,
resp = retry(
put, name,
{'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
resp.read()
self.assertEquals(resp.status, 201)
@ -263,7 +273,8 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEquals(resp.status, 204)
name = uuid4().hex
resp = retry(put, name,
resp = retry(
put, name,
{'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
resp.read()
self.assertEquals(resp.status, 400)
@ -272,7 +283,8 @@ class TestContainer(unittest.TestCase):
self.assertEquals(resp.status, 404)
name = uuid4().hex
resp = retry(put, name,
resp = retry(
put, name,
{'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
resp.read()
self.assertEquals(resp.status, 201)
@ -280,7 +292,8 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEquals(resp.status, 204)
name = uuid4().hex
resp = retry(put, name,
resp = retry(
put, name,
{'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
resp.read()
self.assertEquals(resp.status, 400)
@ -340,25 +353,31 @@ class TestContainer(unittest.TestCase):
def test_POST_bad_metadata(self):
if skip:
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
conn.request('POST', parsed.path + '/' + self.name, '', headers)
return check_response(conn)
resp = retry(post,
resp = retry(
post,
{'X-Container-Meta-' + ('k' * MAX_META_NAME_LENGTH): 'v'})
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(post,
resp = retry(
post,
{'X-Container-Meta-' + ('k' * (MAX_META_NAME_LENGTH + 1)): 'v'})
resp.read()
self.assertEquals(resp.status, 400)
resp = retry(post,
resp = retry(
post,
{'X-Container-Meta-Too-Long': 'k' * MAX_META_VALUE_LENGTH})
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(post,
resp = retry(
post,
{'X-Container-Meta-Too-Long': 'k' * (MAX_META_VALUE_LENGTH + 1)})
resp.read()
self.assertEquals(resp.status, 400)
@ -399,36 +418,42 @@ class TestContainer(unittest.TestCase):
def test_public_container(self):
if skip:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name)
return check_response(conn)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
except Exception as err:
self.assert_(str(err).startswith('No result after '), err)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Read': '.r:*,.rlistings'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(get)
resp.read()
self.assertEquals(resp.status, 204)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': ''})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
except Exception as err:
self.assert_(str(err).startswith('No result after '), err)
def test_cross_account_container(self):
@ -436,27 +461,34 @@ class TestContainer(unittest.TestCase):
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the second account
def get2(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container accessible by the second account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[1],
{'X-Auth-Token': token,
'X-Container-Read': swift_test_perm[1],
'X-Container-Write': swift_test_perm[1]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -464,12 +496,14 @@ class TestContainer(unittest.TestCase):
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 204)
# Make the container private again
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': '',
'X-Container-Write': ''})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -483,27 +517,33 @@ class TestContainer(unittest.TestCase):
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the second account
def get2(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container completely public
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Read': '.r:*,.rlistings'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -511,20 +551,24 @@ class TestContainer(unittest.TestCase):
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 204)
# But we shouldn't be able to write with the second account
def put2(url, token, parsed, conn):
conn.request('PUT', first_account[0] + '/' + self.name + '/object',
'test object', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Now make the container also writeable by the second account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Write': swift_test_perm[1]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -542,26 +586,33 @@ class TestContainer(unittest.TestCase):
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the third account
def get3(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get3, use_account=3)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container accessible by the third account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': swift_test_perm[2]})
{'X-Auth-Token': token,
'X-Container-Read': swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -569,20 +620,24 @@ class TestContainer(unittest.TestCase):
resp = retry(get3, use_account=3)
resp.read()
self.assertEquals(resp.status, 204)
# But we shouldn't be able to write with the third account
def put3(url, token, parsed, conn):
conn.request('PUT', first_account[0] + '/' + self.name + '/object',
'test object', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put3, use_account=3)
resp.read()
self.assertEquals(resp.status, 403)
# Now make the container also writeable by the third account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Write': swift_test_perm[2]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
@ -601,9 +656,10 @@ class TestContainer(unittest.TestCase):
def put(url, token, parsed, conn):
container_name = 'X' * 2048
conn.request('PUT', '%s/%s' % (parsed.path,
container_name), 'there', {'X-Auth-Token': token})
conn.request('PUT', '%s/%s' % (parsed.path, container_name),
'there', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEquals(resp.status, 400)
@ -618,6 +674,7 @@ class TestContainer(unittest.TestCase):
conn.request('PUT', '%s/abc%%00def' % parsed.path, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
if (web_front_end == 'apache2'):
self.assertEquals(resp.status, 404)

173
test/functionalnosetests/test_object.py
View File

@ -1,21 +1,6 @@
#!/usr/bin/python
# Copyright (c) 2010-2013 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2013 Red Hat, Inc.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -55,8 +40,9 @@ class TestObject(unittest.TestCase):
self.obj = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
self.obj), 'test', {'X-Auth-Token': token})
conn.request('PUT', '%s/%s/%s' % (
parsed.path, self.container, self.obj), 'test',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
@ -182,7 +168,7 @@ class TestObject(unittest.TestCase):
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
except Exception as err:
self.assert_(str(err).startswith('No result after '))
def post(url, token, parsed, conn):
@ -207,7 +193,7 @@ class TestObject(unittest.TestCase):
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
except Exception as err:
self.assert_(str(err).startswith('No result after '))
def test_private_object(self):
@ -216,8 +202,8 @@ class TestObject(unittest.TestCase):
# Ensure we can't access the object with the third account
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/%s' % (parsed.path, self.container,
self.obj), '',
conn.request('GET', '%s/%s/%s' % (
parsed.path, self.container, self.obj), '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
@ -228,8 +214,8 @@ class TestObject(unittest.TestCase):
shared_container = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s' % (parsed.path,
shared_container), '',
conn.request('PUT', '%s/%s' % (
parsed.path, shared_container), '',
{'X-Auth-Token': token,
'X-Container-Read': swift_test_perm[2],
'X-Container-Write': swift_test_perm[2]})
@ -240,13 +226,11 @@ class TestObject(unittest.TestCase):
# verify third account can not copy from private container
def copy(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path,
shared_container,
'private_object'),
'', {'X-Auth-Token': token,
conn.request('PUT', '%s/%s/%s' % (
parsed.path, shared_container, 'private_object'), '',
{'X-Auth-Token': token,
'Content-Length': '0',
'X-Copy-From': '%s/%s' % (self.container,
self.obj)})
'X-Copy-From': '%s/%s' % (self.container, self.obj)})
return check_response(conn)
resp = retry(copy, use_account=3)
resp.read()
@ -254,8 +238,9 @@ class TestObject(unittest.TestCase):
# verify third account can write "obj1" to shared container
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, shared_container,
'obj1'), 'test', {'X-Auth-Token': token})
conn.request('PUT', '%s/%s/%s' % (
parsed.path, shared_container, 'obj1'), 'test',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put, use_account=3)
resp.read()
@ -263,12 +248,10 @@ class TestObject(unittest.TestCase):
# verify third account can copy "obj1" to shared container
def copy2(url, token, parsed, conn):
conn.request('COPY', '%s/%s/%s' % (parsed.path,
shared_container,
'obj1'),
'', {'X-Auth-Token': token,
'Destination': '%s/%s' % (shared_container,
'obj1')})
conn.request('COPY', '%s/%s/%s' % (
parsed.path, shared_container, 'obj1'), '',
{'X-Auth-Token': token,
'Destination': '%s/%s' % (shared_container, 'obj1')})
return check_response(conn)
resp = retry(copy2, use_account=3)
resp.read()
@ -276,10 +259,9 @@ class TestObject(unittest.TestCase):
# verify third account STILL can not copy from private container
def copy3(url, token, parsed, conn):
conn.request('COPY', '%s/%s/%s' % (parsed.path,
self.container,
self.obj),
'', {'X-Auth-Token': token,
conn.request('COPY', '%s/%s/%s' % (
parsed.path, self.container, self.obj), '',
{'X-Auth-Token': token,
'Destination': '%s/%s' % (shared_container,
'private_object')})
return check_response(conn)
@ -289,8 +271,9 @@ class TestObject(unittest.TestCase):
# clean up "obj1"
def delete(url, token, parsed, conn):
conn.request('DELETE', '%s/%s/%s' % (parsed.path, shared_container,
'obj1'), '', {'X-Auth-Token': token})
conn.request('DELETE', '%s/%s/%s' % (
parsed.path, shared_container, 'obj1'), '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
@ -316,8 +299,8 @@ class TestObject(unittest.TestCase):
# Upload the first set of segments
def put(url, token, parsed, conn, objnum):
conn.request('PUT', '%s/%s/segments1/%s' % (parsed.path,
self.container, str(objnum)), segments1[objnum],
conn.request('PUT', '%s/%s/segments1/%s' % (
parsed.path, self.container, str(objnum)), segments1[objnum],
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments1)):
@ -327,8 +310,9 @@ class TestObject(unittest.TestCase):
# Upload the manifest
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token,
conn.request('PUT', '%s/%s/manifest' % (
parsed.path, self.container), '', {
'X-Auth-Token': token,
'X-Object-Manifest': '%s/segments1/' % self.container,
'Content-Type': 'text/jibberish', 'Content-Length': '0'})
return check_response(conn)
@ -338,8 +322,8 @@ class TestObject(unittest.TestCase):
# Get the manifest (should get all the segments as the body)
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments1))
@ -348,9 +332,9 @@ class TestObject(unittest.TestCase):
# Get with a range at the start of the second segment
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token, 'Range':
'bytes=3-'})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {
'X-Auth-Token': token, 'Range': 'bytes=3-'})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments1[1:]))
@ -358,9 +342,9 @@ class TestObject(unittest.TestCase):
# Get with a range in the middle of the second segment
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token, 'Range':
'bytes=5-'})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {
'X-Auth-Token': token, 'Range': 'bytes=5-'})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments1)[5:])
@ -368,9 +352,9 @@ class TestObject(unittest.TestCase):
# Get with a full start and stop range
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token, 'Range':
'bytes=5-10'})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {
'X-Auth-Token': token, 'Range': 'bytes=5-10'})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments1)[5:11])
@ -378,8 +362,8 @@ class TestObject(unittest.TestCase):
# Upload the second set of segments
def put(url, token, parsed, conn, objnum):
conn.request('PUT', '%s/%s/segments2/%s' % (parsed.path,
self.container, str(objnum)), segments2[objnum],
conn.request('PUT', '%s/%s/segments2/%s' % (
parsed.path, self.container, str(objnum)), segments2[objnum],
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments2)):
@ -389,8 +373,8 @@ class TestObject(unittest.TestCase):
# Get the manifest (should still be the first segments of course)
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments1))
@ -398,8 +382,9 @@ class TestObject(unittest.TestCase):
# Update the manifest
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token,
conn.request('PUT', '%s/%s/manifest' % (
parsed.path, self.container), '', {
'X-Auth-Token': token,
'X-Object-Manifest': '%s/segments2/' % self.container,
'Content-Length': '0'})
return check_response(conn)
@ -409,8 +394,8 @@ class TestObject(unittest.TestCase):
# Get the manifest (should be the second set of segments now)
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments2))
@ -420,8 +405,8 @@ class TestObject(unittest.TestCase):
# Ensure we can't access the manifest with the third account
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
resp.read()
@ -439,8 +424,8 @@ class TestObject(unittest.TestCase):
# The third account should be able to get the manifest now
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
self.assertEquals(resp.read(), ''.join(segments2))
@ -459,8 +444,8 @@ class TestObject(unittest.TestCase):
# Upload the third set of segments in the other container
def put(url, token, parsed, conn, objnum):
conn.request('PUT', '%s/%s/segments3/%s' % (parsed.path,
acontainer, str(objnum)), segments3[objnum],
conn.request('PUT', '%s/%s/segments3/%s' % (
parsed.path, acontainer, str(objnum)), segments3[objnum],
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments3)):
@ -470,8 +455,9 @@ class TestObject(unittest.TestCase):
# Update the manifest
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token,
conn.request('PUT', '%s/%s/manifest' % (
parsed.path, self.container), '',
{'X-Auth-Token': token,
'X-Object-Manifest': '%s/segments3/' % acontainer,
'Content-Length': '0'})
return check_response(conn)
@ -481,8 +467,8 @@ class TestObject(unittest.TestCase):
# Get the manifest to ensure it's the third set of segments
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get)
self.assertEquals(resp.read(), ''.join(segments3))
@ -495,8 +481,8 @@ class TestObject(unittest.TestCase):
# manifest itself is not).
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
resp.read()
@ -514,8 +500,8 @@ class TestObject(unittest.TestCase):
# The third account should be able to get the manifest now
def get(url, token, parsed, conn):
conn.request('GET', '%s/%s/manifest' % (parsed.path,
self.container), '', {'X-Auth-Token': token})
conn.request('GET', '%s/%s/manifest' % (
parsed.path, self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get, use_account=3)
self.assertEquals(resp.read(), ''.join(segments3))
@ -523,7 +509,8 @@ class TestObject(unittest.TestCase):
# Delete the manifest
def delete(url, token, parsed, conn, objnum):
conn.request('DELETE', '%s/%s/manifest' % (parsed.path,
conn.request('DELETE', '%s/%s/manifest' % (
parsed.path,
self.container), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete, objnum)
@ -532,8 +519,9 @@ class TestObject(unittest.TestCase):
# Delete the third set of segments
def delete(url, token, parsed, conn, objnum):
conn.request('DELETE', '%s/%s/segments3/%s' % (parsed.path,
acontainer, str(objnum)), '', {'X-Auth-Token': token})
conn.request('DELETE', '%s/%s/segments3/%s' % (
parsed.path, acontainer, str(objnum)), '',
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments3)):
resp = retry(delete, objnum)
@ -542,8 +530,9 @@ class TestObject(unittest.TestCase):
# Delete the second set of segments
def delete(url, token, parsed, conn, objnum):
conn.request('DELETE', '%s/%s/segments2/%s' % (parsed.path,
self.container, str(objnum)), '', {'X-Auth-Token': token})
conn.request('DELETE', '%s/%s/segments2/%s' % (
parsed.path, self.container, str(objnum)), '',
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments2)):
resp = retry(delete, objnum)
@ -552,8 +541,9 @@ class TestObject(unittest.TestCase):
# Delete the first set of segments
def delete(url, token, parsed, conn, objnum):
conn.request('DELETE', '%s/%s/segments1/%s' % (parsed.path,
self.container, str(objnum)), '', {'X-Auth-Token': token})
conn.request('DELETE', '%s/%s/segments1/%s' % (
parsed.path, self.container, str(objnum)), '',
{'X-Auth-Token': token})
return check_response(conn)
for objnum in xrange(len(segments1)):
resp = retry(delete, objnum)
@ -574,8 +564,8 @@ class TestObject(unittest.TestCase):
raise SkipTest
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/hi' % (parsed.path,
self.container), 'there', {'X-Auth-Token': token})
conn.request('PUT', '%s/%s/hi' % (parsed.path, self.container),
'there', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
@ -596,7 +586,8 @@ class TestObject(unittest.TestCase):
raise SkipTest
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/abc%%00def' % (parsed.path,
conn.request('PUT', '%s/%s/abc%%00def' % (
parsed.path,
self.container), 'test', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)

89
test/unit/__init__.py
View File

@ -1,3 +1,18 @@
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Swift tests """
import os
@ -8,6 +23,7 @@ from sys import exc_info
from contextlib import contextmanager
from collections import defaultdict
from tempfile import NamedTemporaryFile
import time
from eventlet.green import socket
from tempfile import mkdtemp
from shutil import rmtree
@ -142,13 +158,24 @@ def tmpfile(content):
xattr_data = {}
def _get_inode(fd):
if not isinstance(fd, int):
def _get_inode(fd_or_name):
try:
fd = fd.fileno()
if isinstance(fd_or_name, int):
fd = fd_or_name
else:
try:
fd = fd_or_name.fileno()
except AttributeError:
return os.stat(fd).st_ino
return os.fstat(fd).st_ino
fd = None
if fd is None:
ino = os.stat(fd_or_name).st_ino
else:
ino = os.fstat(fd).st_ino
except OSError as err:
ioerr = IOError()
ioerr.errno = err.errno
raise ioerr
return ino
def _setxattr(fd, k, v):
@ -199,27 +226,57 @@ class NullLoggingHandler(logging.Handler):
pass
class FakeLogger(object):
class UnmockTimeModule(object):
"""
Even if a test mocks time.time - you can restore unmolested behavior in a
another module who imports time directly by monkey patching it's imported
reference to the module with an instance of this class
"""
_orig_time = time.time
def __getattribute__(self, name):
if name == 'time':
return UnmockTimeModule._orig_time
return getattr(time, name)
# logging.LogRecord.__init__ calls time.time
logging.time = UnmockTimeModule()
class FakeLogger(logging.Logger):
# a thread safe logger
def __init__(self, *args, **kwargs):
self._clear()
self.name = 'swift.unit.fake_logger'
self.level = logging.NOTSET
if 'facility' in kwargs:
self.facility = kwargs['facility']
def _clear(self):
self.log_dict = defaultdict(list)
self.lines_dict = defaultdict(list)
def _store_in(store_name):
def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs))
return stub_fn
error = _store_in('error')
info = _store_in('info')
warning = _store_in('warning')
debug = _store_in('debug')
def _store_and_log_in(store_name):
def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs))
self._log(store_name, args[0], args[1:], **kwargs)
return stub_fn
def get_lines_for_level(self, level):
return self.lines_dict[level]
error = _store_and_log_in('error')
info = _store_and_log_in('info')
warning = _store_and_log_in('warning')
debug = _store_and_log_in('debug')
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
@ -267,7 +324,13 @@ class FakeLogger(object):
pass
def handle(self, record):
pass
try:
line = record.getMessage()
except TypeError:
print 'WARNING: unable to format log message %r %% %r' % (
record.msg, record.args)
raise
self.lines_dict[record.levelno].append(line)
def flush(self):
pass
@ -354,7 +417,9 @@ def mock(update):
else:
deletes.append((module, attr))
setattr(module, attr, value)
try:
yield True
finally:
for module, attr, value in returns:
setattr(module, attr, value)
for module, attr in deletes:
@ -466,6 +531,8 @@ def fake_http_connect(*code_iter, **kwargs):
body_iter = iter(body_iter)
def connect(*args, **ckwargs):
if kwargs.get('slow_connect', False):
sleep(0.1)
if 'give_content_type' in kwargs:
if len(args) >= 7 and 'Content-Type' in args[6]:
kwargs['give_content_type'](args[6]['Content-Type'])

53
test/unit/common/test_fs_utils.py
View File

@ -30,8 +30,22 @@ from gluster.swift.common.exceptions import NotDirectoryError, \
def mock_os_fsync(fd):
return True
def mock_tpool_execute(func, *args, **kwargs):
func(*args, **kwargs)
def mock_os_fdatasync(fd):
return True
class TestFakefile(unittest.TestCase):
""" Tests for common.fs_utils.Fake_file """
def test_Fake_file(self):
path = "/tmp/bar"
ff = fs.Fake_file(path)
self.assertEqual(path, ff.path)
self.assertEqual(0, ff.tell())
self.assertEqual(None, ff.read(50))
self.assertEqual(-1, ff.fileno())
self.assertEqual(None, ff.close())
class TestFsUtils(unittest.TestCase):
""" Tests for common.fs_utils """
@ -688,7 +702,6 @@ class TestFsUtils(unittest.TestCase):
fd, tmpfile = mkstemp(dir=tmpdir)
try:
os.write(fd, 'test')
with patch('eventlet.tpool.execute', mock_tpool_execute):
with patch('os.fsync', mock_os_fsync):
assert fs.do_fsync(fd) is None
except GlusterFileSystemOSError as ose:
@ -704,7 +717,6 @@ class TestFsUtils(unittest.TestCase):
try:
fd, tmpfile = mkstemp(dir=tmpdir)
os.write(fd, 'test')
with patch('eventlet.tpool.execute', mock_tpool_execute):
with patch('os.fsync', mock_os_fsync):
assert fs.do_fsync(fd) is None
os.close(fd)
@ -716,3 +728,36 @@ class TestFsUtils(unittest.TestCase):
self.fail("Expected GlusterFileSystemOSError")
finally:
shutil.rmtree(tmpdir)
def test_do_fdatasync(self):
tmpdir = mkdtemp()
try:
fd, tmpfile = mkstemp(dir=tmpdir)
try:
os.write(fd, 'test')
with patch('os.fdatasync', mock_os_fdatasync):
assert fs.do_fdatasync(fd) is None
except GlusterFileSystemOSError as ose:
self.fail('Opening a temporary file failed with %s' %ose.strerror)
else:
os.close(fd)
finally:
shutil.rmtree(tmpdir)
def test_do_fdatasync_err(self):
tmpdir = mkdtemp()
try:
fd, tmpfile = mkstemp(dir=tmpdir)
os.write(fd, 'test')
with patch('os.fdatasync', mock_os_fdatasync):
assert fs.do_fdatasync(fd) is None
os.close(fd)
try:
fs.do_fdatasync(fd)
except GlusterFileSystemOSError:
pass
else:
self.fail("Expected GlusterFileSystemOSError")
finally:
shutil.rmtree(tmpdir)

303
test/unit/obj/test_diskfile.py
View File

@ -43,18 +43,35 @@ from test.unit import FakeLogger
_metadata = {}
def _mock_read_metadata(filename):
def _mapit(filename_or_fd):
if isinstance(filename_or_fd, int):
statmeth = os.fstat
else:
statmeth = os.lstat
try:
stats = statmeth(filename_or_fd)
except OSError as err:
if err.errno == errno.ENOENT:
raise GlusterFileSystemOSError(
err.errno, '%s, os.fstat(%s)' % (err.strerror, filename_or_fd))
raise
return stats.st_ino
def _mock_read_metadata(filename_or_fd):
global _metadata
if filename in _metadata:
md = _metadata[filename]
ino = _mapit(filename_or_fd)
if ino in _metadata:
md = _metadata[ino].copy()
else:
md = {}
return md
def _mock_write_metadata(filename, metadata):
def _mock_write_metadata(filename_or_fd, metadata):
global _metadata
_metadata[filename] = metadata
ino = _mapit(filename_or_fd)
_metadata[ino] = metadata.copy()
def _mock_clear_metadata():
@ -127,8 +144,7 @@ class TestDiskFile(unittest.TestCase):
assert gdf.logger == self.lg
assert gdf.uid == DEFAULT_UID
assert gdf.gid == DEFAULT_GID
assert gdf.metadata == {}
assert gdf.meta_file is None
assert gdf._metadata == None
assert gdf.data_file is None
assert gdf.fp is None
assert gdf.iter_etag is None
@ -146,7 +162,7 @@ class TestDiskFile(unittest.TestCase):
assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "b", "a")
assert gdf.device_path == os.path.join(self.td, "vol0")
def test_constructor_no_metadata(self):
def test_open_no_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
@ -164,12 +180,13 @@ class TestDiskFile(unittest.TestCase):
'Content-Type': 'application/octet-stream'}
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
assert gdf.fp is not None
assert gdf._metadata == exp_md
def test_constructor_existing_metadata(self):
def test_open_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
@ -182,37 +199,39 @@ class TestDiskFile(unittest.TestCase):
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = ini_md
_metadata[_mapit(the_file)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
assert gdf.fp is not None
assert gdf._metadata == exp_md
def test_constructor_invalid_existing_metadata(self):
def test_open_invalid_existing_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = inv_md
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
_metadata[_mapit(the_file)] = inv_md
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata != inv_md
assert gdf.fp is not None
assert gdf._metadata != inv_md
def test_constructor_isdir(self):
def test_open_isdir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
@ -223,29 +242,16 @@ class TestDiskFile(unittest.TestCase):
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_dir] = ini_md
_metadata[_mapit(the_dir)] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
with gdf.open():
assert gdf.data_file == the_dir
assert gdf._is_dir
assert gdf.metadata == exp_md
def test_constructor_keep_data_fp(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z",
keep_data_fp=True)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is not None
assert gdf._metadata == exp_md
def test_constructor_chunk_size(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z",
@ -258,8 +264,7 @@ class TestDiskFile(unittest.TestCase):
assert gdf.iter_hook == 'hook'
def test_close_no_open_fp(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
gdf._is_dir = False
self.called = False
@ -276,28 +281,32 @@ class TestDiskFile(unittest.TestCase):
the_dir = "dir"
self.called = False
os.makedirs(os.path.join(the_cont, the_dir))
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
with gdf.open():
ret = isinstance(gdf.fp, Fake_file)
self.assertTrue(ret)
# Get a File descriptor
fd = gdf.fp
# Get a "Fake_file" pointer
ffp = gdf.fp
# This expected to call Fake_file interfaces
ret = fd.tell()
ret = ffp.tell()
self.assertEqual(ret, 0)
ret = fd.read(1)
self.assertEqual(ret , 0)
ret = ffp.read(1)
self.assertEqual(ret, None)
ret = fd.fileno()
ret = ffp.fileno()
self.assertEqual(ret, -1)
ret = fd.close()
self.assertFalse(self.called)
def our_do_close(ffp):
self.called = True
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
ret = ffp.close()
self.assertEqual(ret, None)
self.assertFalse(self.called)
def test_close_file_object(self):
the_cont = os.path.join(self.td, "vol0", "bar")
@ -306,19 +315,20 @@ class TestDiskFile(unittest.TestCase):
os.makedirs(the_cont)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
def our_do_close(fp):
self.called = True
with mock.patch("gluster.swift.obj.diskfile.do_close",
our_do_close):
gdf.close()
with gdf.open():
assert not self.called
assert self.called
def test_is_deleted(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
with gdf.open():
assert gdf.is_deleted()
gdf.data_file = os.path.join(self.td, "bar")
assert not gdf.is_deleted()
@ -334,7 +344,7 @@ class TestDiskFile(unittest.TestCase):
gdf._create_dir_object(the_dir)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert full_dir_path not in _metadata
assert _mapit(full_dir_path) not in _metadata
def test_create_dir_object_with_md(self):
the_cont = os.path.join(self.td, "vol0", "bar")
@ -349,7 +359,7 @@ class TestDiskFile(unittest.TestCase):
gdf._create_dir_object(the_dir, dir_md)
full_dir_path = os.path.join(the_cont, the_dir)
assert os.path.isdir(full_dir_path)
assert full_dir_path in _metadata
assert _mapit(full_dir_path) in _metadata
def test_create_dir_object_exists(self):
the_path = os.path.join(self.td, "vol0", "bar")
@ -371,7 +381,7 @@ class TestDiskFile(unittest.TestCase):
DiskFileError, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(the_dir in _metadata)
self.assertFalse(_mapit(the_dir) in _metadata)
def test_create_dir_object_do_stat_failure(self):
the_path = os.path.join(self.td, "vol0", "bar")
@ -393,25 +403,26 @@ class TestDiskFile(unittest.TestCase):
DiskFileError, gdf._create_dir_object, the_dir)
gluster.swift.obj.diskfile.do_chown = dc
self.assertFalse(os.path.isdir(the_dir))
self.assertFalse(the_dir in _metadata)
self.assertFalse(_mapit(the_dir) in _metadata)
def test_put_metadata(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "z")
the_dir = os.path.join(self.td, "vol0", "bar", "z")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
md = {'Content-Type': 'application/octet-stream', 'a': 'b'}
gdf.put_metadata(md.copy())
assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (
gdf.metadata, md)
assert _metadata[the_dir] == md
assert gdf._metadata is None
fmd = _metadata[_mapit(the_dir)]
md.update({'X-Object-Type': 'file', 'X-Type': 'Object'})
assert fmd == md, "on-disk md = %r, md = %r" % (fmd, md)
def test_put_w_tombstone(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf.metadata == {}
assert gdf._metadata == None
gdf.put_metadata({'x': '1'}, tombstone=True)
assert gdf.metadata == {}
assert gdf._metadata is None
assert _metadata == {}
def test_put_w_meta_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
@ -420,11 +431,13 @@ class TestDiskFile(unittest.TestCase):
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = gdf.metadata.copy()
with gdf.open():
newmd = gdf.get_metadata().copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
assert gdf._metadata is None
fmd = _metadata[_mapit(the_file)]
assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd)
def test_put_w_meta_file_no_content_type(self):
the_path = os.path.join(self.td, "vol0", "bar")
@ -433,67 +446,72 @@ class TestDiskFile(unittest.TestCase):
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
newmd = gdf.metadata.copy()
with gdf.open():
newmd = gdf.get_metadata().copy()
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
assert gdf._metadata is None
fmd = _metadata[_mapit(the_file)]
assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd)
def test_put_w_meta_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = gdf.metadata.copy()
with gdf.open():
newmd = gdf.get_metadata().copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
assert gdf._metadata is None
fmd = _metadata[_mapit(the_dir)]
assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd)
def test_put_w_marker_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
newmd = gdf.metadata.copy()
with gdf.open():
newmd = gdf.get_metadata().copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
assert gdf._metadata is None
fmd = _metadata[_mapit(the_dir)]
assert fmd == newmd, "on-disk md = %r, newmd = %r" % (fmd, newmd)
def test_put_w_marker_dir_create(self):
the_cont = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_cont, "dir")
os.makedirs(the_cont)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
assert gdf.metadata == {}
assert gdf._metadata == None
newmd = {
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'}
with gdf.writer() as dw:
dw.put(newmd, extension='.dir')
with gdf.create() as dw:
dw.put(newmd.copy(), extension='.dir')
with gdf.open():
assert gdf.data_file == the_dir
for key, val in newmd.items():
assert gdf.metadata[key] == val
assert _metadata[the_dir][key] == val
assert gdf.metadata[X_OBJECT_TYPE] == DIR_OBJECT
assert _metadata[the_dir][X_OBJECT_TYPE] == DIR_OBJECT
assert gdf._metadata[key] == val
assert _metadata[_mapit(the_dir)][key] == val
assert X_OBJECT_TYPE not in gdf._metadata, "md = %r" % gdf._metadata
assert _metadata[_mapit(the_dir)][X_OBJECT_TYPE] == DIR_OBJECT
def test_put_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir")
origmd = gdf.metadata.copy()
origfmd = _metadata[the_dir]
newmd = gdf.metadata.copy()
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
with gdf.writer() as dw:
newmd = {
'Content-Type': '',
'X-Object-Meta-test': '1234'}
with gdf.create() as dw:
try:
dw.put(newmd, extension='.data')
except DiskFileError:
@ -501,8 +519,6 @@ class TestDiskFile(unittest.TestCase):
else:
self.fail("Expected to encounter"
" 'already-exists-as-dir' exception")
assert gdf.metadata == origmd
assert _metadata[the_dir] == origfmd
def test_put(self):
the_cont = os.path.join(self.td, "vol0", "bar")
@ -525,7 +541,7 @@ class TestDiskFile(unittest.TestCase):
'Content-Length': '5',
}
with gdf.writer() as dw:
with gdf.create() as dw:
assert dw.tmppath is not None
tmppath = dw.tmppath
dw.write(body)
@ -561,7 +577,7 @@ class TestDiskFile(unittest.TestCase):
with mock.patch("os.open", mock_open):
try:
with gdf.writer() as dw:
with gdf.create() as dw:
assert dw.tmppath is not None
dw.write(body)
dw.put(metadata)
@ -601,7 +617,7 @@ class TestDiskFile(unittest.TestCase):
with mock.patch("gluster.swift.obj.diskfile.sleep", mock_sleep):
with mock.patch("os.rename", mock_rename):
try:
with gdf.writer() as dw:
with gdf.create() as dw:
assert dw.tmppath is not None
dw.write(body)
dw.put(metadata)
@ -631,7 +647,7 @@ class TestDiskFile(unittest.TestCase):
'Content-Length': '5',
}
with gdf.writer() as dw:
with gdf.create() as dw:
assert dw.tmppath is not None
tmppath = dw.tmppath
dw.write(body)
@ -642,32 +658,32 @@ class TestDiskFile(unittest.TestCase):
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
def test_unlinkold_no_metadata(self):
def test_delete_no_metadata(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf.metadata == {}
assert gdf._metadata == None
_saved_rmobjdir = gluster.swift.obj.diskfile.rmobjdir
gluster.swift.obj.diskfile.rmobjdir = _mock_rmobjdir
try:
gdf.unlinkold(None)
gdf.delete(1.0)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.obj.diskfile.rmobjdir = _saved_rmobjdir
def test_unlinkold_same_timestamp(self):
def test_delete_same_timestamp(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf.metadata == {}
gdf.metadata['X-Timestamp'] = 1
assert gdf._metadata == None
gdf._metadata = {'X-Timestamp': 1}
_saved_rmobjdir = gluster.swift.obj.diskfile.rmobjdir
gluster.swift.obj.diskfile.rmobjdir = _mock_rmobjdir
try:
gdf.unlinkold(1)
gdf.delete(1)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.obj.diskfile.rmobjdir = _saved_rmobjdir
def test_unlinkold_file(self):
def test_delete_file(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
@ -675,15 +691,14 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
later = float(gdf.get_metadata()['X-Timestamp']) + 1
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
def test_unlinkold_file_not_found(self):
def test_delete_file_not_found(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
@ -691,18 +706,19 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
later = float(gdf._metadata['X-Timestamp']) + 1
assert gdf.data_file == the_file
assert not gdf._is_dir
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
def test_unlinkold_file_unlink_error(self):
def test_delete_file_unlink_error(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
os.makedirs(the_path)
@ -710,10 +726,10 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
later = float(gdf._metadata['X-Timestamp']) + 1
def _mock_os_unlink_eacces_err(f):
raise OSError(errno.EACCES, os.strerror(errno.EACCES))
@ -725,7 +741,7 @@ class TestDiskFile(unittest.TestCase):
# Handle the case os_unlink() raises an OSError
with patch("os.unlink", _mock_os_unlink_eacces_err):
try:
gdf.unlinkold(normalize_timestamp(later))
gdf.delete(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
@ -736,17 +752,17 @@ class TestDiskFile(unittest.TestCase):
assert os.path.isdir(gdf.datadir)
assert os.path.exists(os.path.join(gdf.datadir, gdf._obj))
def test_unlinkold_is_dir(self):
def test_delete_is_dir(self):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
with gdf.open():
assert gdf.data_file == the_dir
assert gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
later = float(gdf._metadata['X-Timestamp']) + 1
gdf.delete(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
@ -758,6 +774,7 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
assert 4 == gdf.get_data_file_size()
@ -770,12 +787,13 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
assert 4 == gdf.metadata['Content-Length']
gdf.metadata['Content-Length'] = 3
assert 4 == gdf._metadata['Content-Length']
gdf._metadata['Content-Length'] = 3
assert 4 == gdf.get_data_file_size()
assert 4 == gdf.metadata['Content-Length']
assert 4 == gdf._metadata['Content-Length']
def test_get_data_file_size_dne(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar",
@ -795,6 +813,7 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
gdf.data_file = gdf.data_file + ".dne"
@ -813,6 +832,7 @@ class TestDiskFile(unittest.TestCase):
fd.write("1234")
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf._obj == "z"
with gdf.open():
assert gdf.data_file == the_file
assert not gdf._is_dir
stats = os.stat(the_path)
@ -836,32 +856,33 @@ class TestDiskFile(unittest.TestCase):
the_path = os.path.join(self.td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
os.makedirs(the_dir)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d",
keep_data_fp=True)
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "d")
assert gdf._obj == "d"
with gdf.open():
assert gdf.data_file == the_dir
assert gdf._is_dir
assert 0 == gdf.get_data_file_size()
def test_filter_metadata(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "z")
assert gdf.metadata == {}
assert gdf._metadata == None
gdf._filter_metadata()
assert gdf.metadata == {}
assert gdf._metadata == None
gdf.metadata[X_TYPE] = 'a'
gdf.metadata[X_OBJECT_TYPE] = 'b'
gdf.metadata['foobar'] = 'c'
gdf._metadata = {}
gdf._metadata[X_TYPE] = 'a'
gdf._metadata[X_OBJECT_TYPE] = 'b'
gdf._metadata['foobar'] = 'c'
gdf._filter_metadata()
assert X_TYPE not in gdf.metadata
assert X_OBJECT_TYPE not in gdf.metadata
assert 'foobar' in gdf.metadata
assert X_TYPE not in gdf._metadata
assert X_OBJECT_TYPE not in gdf._metadata
assert 'foobar' in gdf._metadata
def test_writer(self):
def test_create(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
saved_fd = None
with gdf.writer() as dw:
with gdf.create() as dw:
assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath
@ -881,10 +902,10 @@ class TestDiskFile(unittest.TestCase):
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
def test_writer_err_on_close(self):
def test_create_err_on_close(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.writer() as dw:
with gdf.create() as dw:
assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath
@ -896,10 +917,10 @@ class TestDiskFile(unittest.TestCase):
os.close(dw.fd)
assert not os.path.exists(saved_tmppath)
def test_writer_err_on_unlink(self):
def test_create_err_on_unlink(self):
gdf = self._get_diskfile("vol0", "p57", "ufo47", "bar", "dir/z")
saved_tmppath = ''
with gdf.writer() as dw:
with gdf.create() as dw:
assert gdf.datadir == os.path.join(self.td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = dw.tmppath

24
test/unit/proxy/controllers/test_account.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -40,6 +40,28 @@ class TestAccountController(unittest.TestCase):
self.assertEqual(headers_to_account_info(resp.headers),
resp.environ['swift.account/AUTH_bob'])
def test_swift_owner(self):
owner_headers = {
'x-account-meta-temp-url-key': 'value',
'x-account-meta-temp-url-key-2': 'value'}
controller = proxy_server.AccountController(self.app, 'a')
req = Request.blank('/a')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, headers=owner_headers)):
resp = controller.HEAD(req)
self.assertEquals(2, resp.status_int // 100)
for key in owner_headers:
self.assertTrue(key not in resp.headers)
req = Request.blank('/a', environ={'swift_owner': True})
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, headers=owner_headers)):
resp = controller.HEAD(req)
self.assertEquals(2, resp.status_int // 100)
for key in owner_headers:
self.assertTrue(key in resp.headers)
if __name__ == '__main__':
unittest.main()

142
test/unit/proxy/controllers/test_base.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,9 +16,9 @@
import unittest
from mock import patch
from swift.proxy.controllers.base import headers_to_container_info, \
headers_to_account_info, get_container_info, get_container_memcache_key, \
get_account_info, get_account_memcache_key, _get_cache_key, get_info, \
Controller
headers_to_account_info, headers_to_object_info, get_container_info, \
get_container_memcache_key, get_account_info, get_account_memcache_key, \
get_object_env_key, _get_cache_key, get_info, get_object_info, Controller
from swift.common.swob import Request
from swift.common.utils import split_path
from test.unit import fake_http_connect, FakeRing, FakeMemcache
@ -29,12 +29,18 @@ FakeResponse_status_int = 201
class FakeResponse(object):
def __init__(self, headers, env, account, container):
def __init__(self, headers, env, account, container, obj):
self.headers = headers
self.status_int = FakeResponse_status_int
self.environ = env
if obj:
env_key = get_object_env_key(account, container, obj)
else:
cache_key, env_key = _get_cache_key(account, container)
if container:
if account and container and obj:
info = headers_to_object_info(headers, FakeResponse_status_int)
elif account and container:
info = headers_to_container_info(headers, FakeResponse_status_int)
else:
info = headers_to_account_info(headers, FakeResponse_status_int)
@ -42,18 +48,27 @@ class FakeResponse(object):
class FakeRequest(object):
def __init__(self, env, path):
def __init__(self, env, path, swift_source=None):
self.environ = env
(version, account, container, obj) = split_path(path, 2, 4, True)
self.account = account
self.container = container
self.obj = obj
if obj:
stype = 'object'
self.headers = {'content-length': 5555,
'content-type': 'text/plain'}
else:
stype = container and 'container' or 'account'
self.headers = {'x-%s-object-count' % (stype): 1000,
'x-%s-bytes-used' % (stype): 6666}
if swift_source:
meta = 'x-%s-meta-fakerequest-swift-source' % stype
self.headers[meta] = swift_source
def get_response(self, app):
return FakeResponse(self.headers, self.environ, self.account,
self.container)
self.container, self.obj)
class FakeCache(object):
@ -73,6 +88,21 @@ class TestFuncs(unittest.TestCase):
def test_GETorHEAD_base(self):
base = Controller(self.app)
req = Request.blank('/a/c/o/with/slashes')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
'/a/c/o/with/slashes')
self.assertTrue('swift.object/a/c/o/with/slashes' in resp.environ)
self.assertEqual(
resp.environ['swift.object/a/c/o/with/slashes']['status'], 200)
req = Request.blank('/a/c/o')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
resp = base.GETorHEAD_base(req, 'object', FakeRing(), 'part',
'/a/c/o')
self.assertTrue('swift.object/a/c/o' in resp.environ)
self.assertEqual(resp.environ['swift.object/a/c/o']['status'], 200)
req = Request.blank('/a/c')
with patch('swift.proxy.controllers.base.'
'http_connect', fake_http_connect(200)):
@ -101,7 +131,7 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
self.assertEquals(env.get('swift.account/a'), info_a)
# Do an env cached call to account
info_a = get_info(None, env, 'a')
@ -110,7 +140,7 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
self.assertEquals(env.get('swift.account/a'), info_a)
# This time do env cached call to account and non cached to container
with patch('swift.proxy.controllers.base.'
@ -121,10 +151,11 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
self.assertEquals(env.get('swift.account/a'), info_a)
self.assertEquals(env.get('swift.container/a/c'), info_c)
# This time do a non cached call to account than non cached to container
# This time do a non cached call to account than non cached to
# container
env = {} # abandon previous call to env
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
@ -134,10 +165,11 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env['swift.account/a'], info_a)
self.assertEquals(env['swift.container/a/c'], info_c)
self.assertEquals(env.get('swift.account/a'), info_a)
self.assertEquals(env.get('swift.container/a/c'), info_c)
# This time do an env cached call to container while account is not cached
# This time do an env cached call to container while account is not
# cached
del(env['swift.account/a'])
info_c = get_info(None, env, 'a', 'c')
# Check that you got proper info
@ -145,7 +177,7 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_c['bytes'], 6666)
self.assertEquals(info_c['object_count'], 1000)
# Make sure the env cache is set and account still not cached
self.assertEquals(env, {'swift.container/a/c': info_c})
self.assertEquals(env.get('swift.container/a/c'), info_c)
# Do a non cached call to account not found with ret_not_found
env = {}
@ -161,7 +193,7 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
self.assertEquals(env.get('swift.account/a'), info_a)
# Do a cached call to account not found with ret_not_found
info_a = get_info(None, env, 'a', ret_not_found=True)
@ -170,7 +202,7 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_a['bytes'], 6666)
self.assertEquals(info_a['total_object_count'], 1000)
# Make sure the env cache is set
self.assertEquals(env, {'swift.account/a': info_a})
self.assertEquals(env.get('swift.account/a'), info_a)
# Do a non cached call to account not found without ret_not_found
env = {}
@ -191,6 +223,21 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(info_a, None)
self.assertEquals(env['swift.account/a']['status'], 404)
def test_get_container_info_swift_source(self):
req = Request.blank("/v1/a/c", environ={'swift.cache': FakeCache({})})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_container_info(req.environ, 'app', swift_source='MC')
self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC')
def test_get_object_info_swift_source(self):
req = Request.blank("/v1/a/c/o",
environ={'swift.cache': FakeCache({})})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_object_info(req.environ, 'app', swift_source='LU')
self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU')
def test_get_container_info_no_cache(self):
req = Request.blank("/v1/AUTH_account/cont",
environ={'swift.cache': FakeCache({})})
@ -222,6 +269,13 @@ class TestFuncs(unittest.TestCase):
resp = get_container_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_account_info_swift_source(self):
req = Request.blank("/v1/a", environ={'swift.cache': FakeCache({})})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_account_info(req.environ, 'a', swift_source='MC')
self.assertEquals(resp['meta']['fakerequest-swift-source'], 'MC')
def test_get_account_info_no_cache(self):
req = Request.blank("/v1/AUTH_account",
environ={'swift.cache': FakeCache({})})
@ -271,6 +325,28 @@ class TestFuncs(unittest.TestCase):
resp = get_account_info(req.environ, 'xxx')
self.assertEquals(resp['bytes'], 3867)
def test_get_object_info_env(self):
cached = {'status': 200,
'length': 3333,
'type': 'application/json',
'meta': {}}
env_key = get_object_env_key("account", "cont", "obj")
req = Request.blank("/v1/account/cont/obj",
environ={env_key: cached,
'swift.cache': FakeCache({})})
resp = get_object_info(req.environ, 'xxx')
self.assertEquals(resp['length'], 3333)
self.assertEquals(resp['type'], 'application/json')
def test_get_object_info_no_env(self):
req = Request.blank("/v1/account/cont/obj",
environ={'swift.cache': FakeCache({})})
with patch('swift.proxy.controllers.base.'
'_prepare_pre_auth_info_request', FakeRequest):
resp = get_object_info(req.environ, 'xxx')
self.assertEquals(resp['length'], 5555)
self.assertEquals(resp['type'], 'text/plain')
def test_headers_to_container_info_missing(self):
resp = headers_to_container_info({}, 404)
self.assertEquals(resp['status'], 404)
@ -329,3 +405,31 @@ class TestFuncs(unittest.TestCase):
self.assertEquals(
resp,
headers_to_account_info(headers.items(), 200))
def test_headers_to_object_info_missing(self):
resp = headers_to_object_info({}, 404)
self.assertEquals(resp['status'], 404)
self.assertEquals(resp['length'], None)
self.assertEquals(resp['etag'], None)
def test_headers_to_object_info_meta(self):
headers = {'X-Object-Meta-Whatevs': 14,
'x-object-meta-somethingelse': 0}
resp = headers_to_object_info(headers.items(), 200)
self.assertEquals(len(resp['meta']), 2)
self.assertEquals(resp['meta']['whatevs'], 14)
self.assertEquals(resp['meta']['somethingelse'], 0)
def test_headers_to_object_info_values(self):
headers = {
'content-length': '1024',
'content-type': 'application/json',
}
resp = headers_to_object_info(headers.items(), 200)
self.assertEquals(resp['length'], '1024')
self.assertEquals(resp['type'], 'application/json')
headers['x-unused-header'] = 'blahblahblah'
self.assertEquals(
resp,
headers_to_object_info(headers.items(), 200))

24
test/unit/proxy/controllers/test_container.py
View File

@ -1,4 +1,4 @@
# Copyright (c) 2010-2012 OpenStack, LLC.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -40,6 +40,28 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(headers_to_container_info(resp.headers),
resp.environ['swift.container/a/c'])
def test_swift_owner(self):
owner_headers = {
'x-container-read': 'value', 'x-container-write': 'value',
'x-container-sync-key': 'value', 'x-container-sync-to': 'value'}
controller = proxy_server.ContainerController(self.app, 'a', 'c')
req = Request.blank('/a/c')
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, headers=owner_headers)):
resp = controller.HEAD(req)
self.assertEquals(2, resp.status_int // 100)
for key in owner_headers:
self.assertTrue(key not in resp.headers)
req = Request.blank('/a/c', environ={'swift_owner': True})
with mock.patch('swift.proxy.controllers.base.http_connect',
fake_http_connect(200, 200, headers=owner_headers)):
resp = controller.HEAD(req)
self.assertEquals(2, resp.status_int // 100)
for key in owner_headers:
self.assertTrue(key in resp.headers)
if __name__ == '__main__':
unittest.main()

64
test/unit/proxy/controllers/test_obj.py
View File

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright (c) 2010-2012 OpenStack, LLC.
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,9 +15,28 @@
# limitations under the License.
import unittest
from contextlib import contextmanager
import mock
import swift
from swift.proxy import server as proxy_server
from test.unit import FakeRing, FakeMemcache
from test.unit import FakeRing, FakeMemcache, fake_http_connect
@contextmanager
def set_http_connect(*args, **kwargs):
old_connect = swift.proxy.controllers.base.http_connect
new_connect = fake_http_connect(*args, **kwargs)
swift.proxy.controllers.base.http_connect = new_connect
swift.proxy.controllers.obj.http_connect = new_connect
swift.proxy.controllers.account.http_connect = new_connect
swift.proxy.controllers.container.http_connect = new_connect
yield new_connect
swift.proxy.controllers.base.http_connect = old_connect
swift.proxy.controllers.obj.http_connect = old_connect
swift.proxy.controllers.account.http_connect = old_connect
swift.proxy.controllers.container.http_connect = old_connect
class TestObjControllerWriteAffinity(unittest.TestCase):
@ -44,7 +63,8 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
def test_iter_nodes_local_first_moves_locals_first(self):
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
self.app.write_affinity_is_local_fn = (lambda node: node['region'] == 1)
self.app.write_affinity_is_local_fn = (
lambda node: node['region'] == 1)
self.app.write_affinity_node_count = lambda ring: 4
all_nodes = self.app.object_ring.get_part_nodes(1)
@ -59,6 +79,44 @@ class TestObjControllerWriteAffinity(unittest.TestCase):
# we don't skip any nodes
self.assertEqual(sorted(all_nodes), sorted(local_first_nodes))
def test_connect_put_node_timeout(self):
controller = proxy_server.ObjectController(self.app, 'a', 'c', 'o')
self.app.conn_timeout = 0.1
with set_http_connect(200, slow_connect=True):
nodes = [dict(ip='', port='', device='')]
res = controller._connect_put_node(nodes, '', '', {}, ('', ''))
self.assertTrue(res is None)
class TestObjController(unittest.TestCase):
def test_PUT_log_info(self):
# mock out enough to get to the area of the code we want to test
with mock.patch('swift.proxy.controllers.obj.check_object_creation',
mock.MagicMock(return_value=None)):
app = mock.MagicMock()
app.container_ring.get_nodes.return_value = (1, [2])
app.object_ring.get_nodes.return_value = (1, [2])
controller = proxy_server.ObjectController(app, 'a', 'c', 'o')
controller.container_info = mock.MagicMock(return_value={
'partition': 1,
'nodes': [{}],
'write_acl': None,
'sync_key': None,
'versions': None})
# and now test that we add the header to log_info
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.headers['x-copy-from'] = 'somewhere'
controller.PUT(req)
self.assertEquals(
req.environ.get('swift.log_info'), ['x-copy-from:somewhere'])
# and then check that we don't do that for originating POSTs
req = swift.common.swob.Request.blank('/v1/a/c/o')
req.method = 'POST'
req.headers['x-copy-from'] = 'elsewhere'
controller.PUT(req)
self.assertEquals(req.environ.get('swift.log_info'), None)
if __name__ == '__main__':
unittest.main()

742
test/unit/proxy/test_server.py
View File

File diff suppressed because it is too large Load Diff

2
tox.ini
View File

@ -10,7 +10,7 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps =
https://launchpad.net/swift/havana/1.9.1/+download/swift-1.9.1.tar.gz
https://launchpad.net/swift/havana/1.10.0/+download/swift-1.10.0.tar.gz
--download-cache={homedir}/.pipcache
-r{toxinidir}/tools/test-requires
changedir = {toxinidir}/test/unit