openstack/swift
Code Issues Proposed changes

merged with trunk

Browse Source
This commit is contained in:
John Dickinson 2010-09-10 15:09:41 -05:00
commit 803f26c306
41 changed files with 2895 additions and 858 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

468
bin/st
View File

@ -154,19 +154,26 @@ except:
def get_auth(url, user, key, snet=False):
"""
Get authentication credentials
Get authentication/authorization credentials.
:param url: authentication URL
:param user: user to auth as
:param key: key or passowrd for auth
:param snet: use SERVICENET internal network default is False
:returns: tuple of (storage URL, storage token, auth token)
The snet parameter is used for Rackspace's ServiceNet internal network
implementation. In this function, it simply adds *snet-* to the beginning
of the host name for the returned storage URL. With Rackspace Cloud Files,
use of this network path causes no bandwidth charges but requires the
client to be running on Rackspace's ServiceNet network.
:param url: authentication/authorization URL
:param user: user to authenticate as
:param key: key or password for authorization
:param snet: use SERVICENET internal network (see above), default is False
:returns: tuple of (storage URL, auth token)
:raises ClientException: HTTP GET request to auth URL failed
"""
parsed, conn = http_connection(url)
conn.request('GET', parsed.path, '',
{'X-Auth-User': user, 'X-Auth-Key': key})
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
@ -175,7 +182,7 @@ except:
url = resp.getheader('x-storage-url')
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
url = urlunparse(parsed)
return url, resp.getheader('x-storage-token',
@ -196,18 +203,21 @@ except:
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of accounts
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_account(url, token, marker, limit, prefix, http_conn)
rv = get_account(url, token, marker, limit, prefix, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
marker = listing[-1]['name']
listing = get_account(url, token, marker, limit, prefix, http_conn)
listing = \
get_account(url, token, marker, limit, prefix, http_conn)[1]
if listing:
rv.extend(listing)
return rv
parsed, conn = http_conn
qs = 'format=json'
@ -220,6 +230,9 @@ except:
conn.request('GET', '%s?%s' % (parsed.path, qs), '',
{'X-Auth-Token': token})
resp = conn.getresponse()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@ -228,8 +241,8 @@ except:
http_reason=resp.reason)
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_account(url, token, http_conn=None):
@ -240,7 +253,8 @@ except:
:param token: auth token
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (container count, object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -249,14 +263,42 @@ except:
parsed, conn = http_connection(url)
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-account-container-count', 0)), \
int(resp.getheader('x-account-object-count', 0)), \
int(resp.getheader('x-account-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def post_account(url, token, headers, http_conn=None):
"""
Update an account's metadata.
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
headers['X-Auth-Token'] = token
conn.request('POST', parsed.path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def get_container(url, token, container, marker=None, limit=None,
@ -276,23 +318,25 @@ except:
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, http_conn)
prefix, delimiter, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
path = '%s/%s' % (parsed.path, quote(container))
@ -313,10 +357,13 @@ except:
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_query=qs,
http_status=resp.status, http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None):
@ -328,7 +375,8 @@ except:
:param container: container name to get stats for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -344,17 +392,20 @@ except:
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count', 0)), \
int(resp.getheader('x-container-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_container(url, token, container, http_conn=None):
def put_container(url, token, container, headers=None, http_conn=None):
"""
Create a container
:param url: storage URL
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP PUT request failed
@ -364,7 +415,10 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('PUT', path, '', {'X-Auth-Token': token})
if not headers:
headers = {}
headers['X-Auth-Token'] = token
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
@ -374,6 +428,34 @@ except:
http_reason=resp.reason)
def post_container(url, token, container, headers, http_conn=None):
"""
Update a container's metadata.
:param url: storage URL
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def delete_container(url, token, container, http_conn=None):
"""
Delete a container
@ -411,8 +493,12 @@ except:
:param name: object name to get
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param resp_chunk_size: if defined, chunk size of data to read
:returns: a list of objects
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
request.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if http_conn:
@ -427,10 +513,6 @@ except:
raise ClientException('Object GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
if resp_chunk_size:
def _object_body():
@ -441,12 +523,10 @@ except:
object_body = _object_body()
else:
object_body = resp.read()
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata, \
object_body
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def head_object(url, token, container, name, http_conn=None):
@ -459,8 +539,8 @@ except:
:param name: object name to get info for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (content type, content length, last modfied, etag,
dictionary of metadata)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -475,20 +555,15 @@ except:
raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_object(url, token, container, name, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None, http_conn=None):
def put_object(url, token, container, name, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None, headers=None,
http_conn=None):
"""
Put an object
@ -496,12 +571,12 @@ except:
:param token: auth token
:param container: container name that the object is in
:param name: object name to put
:param contents: file like object to read object data from
:param metadata: dictionary of object metadata
:param contents: a string or a file like object to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param chunk_size: chunk size of data to write
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: etag from server response
@ -512,9 +587,9 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
if not headers:
headers = {}
headers['X-Auth-Token'] = token
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
@ -550,15 +625,15 @@ except:
return resp.getheader('etag').strip('"')
def post_object(url, token, container, name, metadata, http_conn=None):
def post_object(url, token, container, name, headers, http_conn=None):
"""
Change object metadata
Update object metadata
:param url: storage URL
:param token: auth token
:param container: container name that the object is in
:param name: object name to change
:param metadata: dictionary of object metadata
:param name: name of the object to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
@ -568,9 +643,7 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
@ -620,7 +693,7 @@ except:
:param preauthurl: storage URL (if you have already authenticated)
:param preauthtoken: authentication token (if you have already
authenticated)
:param snet: use SERVICENET internal network default is False
:param snet: use SERVICENET internal network default is False
"""
self.authurl = authurl
self.user = user
@ -632,20 +705,24 @@ except:
self.attempts = 0
self.snet = snet
def get_auth(self):
return get_auth(self.authurl, self.user, self.key, snet=self.snet)
def http_connection(self):
return http_connection(self.url)
def _retry(self, func, *args, **kwargs):
kwargs['http_conn'] = self.http_conn
self.attempts = 0
backoff = 1
while self.attempts <= self.retries:
self.attempts += 1
try:
if not self.url or not self.token:
self.url, self.token = \
get_auth(self.authurl, self.user, self.key, snet=self.snet)
self.url, self.token = self.get_auth()
self.http_conn = None
if not self.http_conn:
self.http_conn = http_connection(self.url)
kwargs['http_conn'] = self.http_conn
self.http_conn = self.http_connection()
kwargs['http_conn'] = self.http_conn
rv = func(self.url, self.token, *args, **kwargs)
return rv
except (socket.error, HTTPException):
@ -667,63 +744,71 @@ except:
backoff *= 2
def head_account(self):
"""Wrapper for head_account"""
"""Wrapper for :func:`head_account`"""
return self._retry(head_account)
def get_account(self, marker=None, limit=None, prefix=None,
full_listing=False):
"""Wrapper for get_account"""
# TODO: With full_listing=True this will restart the entire listing
# with each retry. Need to make a better version that just retries
# where it left off.
"""Wrapper for :func:`get_account`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(get_account, marker=marker, limit=limit,
prefix=prefix, full_listing=full_listing)
def post_account(self, headers):
"""Wrapper for :func:`post_account`"""
return self._retry(post_account, headers)
def head_container(self, container):
"""Wrapper for head_container"""
"""Wrapper for :func:`head_container`"""
return self._retry(head_container, container)
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, full_listing=False):
"""Wrapper for get_container"""
# TODO: With full_listing=True this will restart the entire listing
# with each retry. Need to make a better version that just retries
# where it left off.
"""Wrapper for :func:`get_container`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(get_container, container, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
def put_container(self, container):
"""Wrapper for put_container"""
return self._retry(put_container, container)
def put_container(self, container, headers=None):
"""Wrapper for :func:`put_container`"""
return self._retry(put_container, container, headers=headers)
def post_container(self, container, headers):
"""Wrapper for :func:`post_container`"""
return self._retry(post_container, container, headers)
def delete_container(self, container):
"""Wrapper for delete_container"""
"""Wrapper for :func:`delete_container`"""
return self._retry(delete_container, container)
def head_object(self, container, obj):
"""Wrapper for head_object"""
"""Wrapper for :func:`head_object`"""
return self._retry(head_object, container, obj)
def get_object(self, container, obj, resp_chunk_size=None):
"""Wrapper for get_object"""
"""Wrapper for :func:`get_object`"""
return self._retry(get_object, container, obj,
resp_chunk_size=resp_chunk_size)
def put_object(self, container, obj, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None):
"""Wrapper for put_object"""
def put_object(self, container, obj, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None,
headers=None):
"""Wrapper for :func:`put_object`"""
return self._retry(put_object, container, obj, contents,
metadata=metadata, content_length=content_length, etag=etag,
chunk_size=chunk_size, content_type=content_type)
content_length=content_length, etag=etag, chunk_size=chunk_size,
content_type=content_type, headers=headers)
def post_object(self, container, obj, metadata):
"""Wrapper for post_object"""
return self._retry(post_object, container, obj, metadata)
def post_object(self, container, obj, headers):
"""Wrapper for :func:`post_object`"""
return self._retry(post_object, container, obj, headers)
def delete_object(self, container, obj):
"""Wrapper for delete_object"""
"""Wrapper for :func:`delete_object`"""
return self._retry(delete_object, container, obj)
# End inclusion of swift.common.client
@ -805,7 +890,7 @@ def st_delete(options, args):
marker = ''
while True:
objects = [o['name'] for o in
conn.get_container(container, marker=marker)]
conn.get_container(container, marker=marker)[1]]
if not objects:
break
for obj in objects:
@ -847,7 +932,7 @@ def st_delete(options, args):
marker = ''
while True:
containers = \
[c['name'] for c in conn.get_account(marker=marker)]
[c['name'] for c in conn.get_account(marker=marker)[1]]
if not containers:
break
for container in containers:
@ -893,8 +978,11 @@ def st_download(options, args):
object_queue = Queue(10000)
def _download_object((container, obj), conn):
try:
content_type, content_length, _, etag, metadata, body = \
headers, body = \
conn.get_object(container, obj, resp_chunk_size=65536)
content_type = headers.get('content-type')
content_length = int(headers.get('content-length'))
etag = headers.get('etag')
path = options.yes_all and join(container, obj) or obj
if path[:1] in ('/', '\\'):
path = path[1:]
@ -925,8 +1013,8 @@ def st_download(options, args):
options.error_queue.put(
'%s: read_length != content_length, %d != %d' %
(path, read_length, content_length))
if 'mtime' in metadata:
mtime = float(metadata['mtime'])
if 'x-object-meta-mtime' in headers:
mtime = float(headers['x-object-meta-mtime'])
utime(path, (mtime, mtime))
if options.verbose:
options.print_queue.put(path)
@ -941,7 +1029,7 @@ def st_download(options, args):
marker = ''
while True:
objects = [o['name'] for o in
conn.get_container(container, marker=marker)]
conn.get_container(container, marker=marker)[1]]
if not objects:
break
for obj in objects:
@ -969,7 +1057,7 @@ def st_download(options, args):
marker = ''
while True:
containers = [c['name']
for c in conn.get_account(marker=marker)]
for c in conn.get_account(marker=marker)[1]]
if not containers:
break
for container in containers:
@ -1016,10 +1104,11 @@ def st_list(options, args):
marker = ''
while True:
if not args:
items = conn.get_account(marker=marker, prefix=options.prefix)
items = \
conn.get_account(marker=marker, prefix=options.prefix)[1]
else:
items = conn.get_container(args[0], marker=marker,
prefix=options.prefix, delimiter=options.delimiter)
prefix=options.prefix, delimiter=options.delimiter)[1]
if not items:
break
for item in items:
@ -1042,46 +1131,85 @@ def st_stat(options, args):
conn = Connection(options.auth, options.user, options.key)
if not args:
try:
container_count, object_count, bytes_used = conn.head_account()
headers = conn.head_account()
container_count = int(headers.get('x-account-container-count', 0))
object_count = int(headers.get('x-account-object-count', 0))
bytes_used = int(headers.get('x-account-bytes-used', 0))
options.print_queue.put('''
Account: %s
Containers: %d
Objects: %d
Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count,
object_count, bytes_used))
for key, value in headers.items():
if key.startswith('x-account-meta-'):
options.print_queue.put('%10s: %s' % ('Meta %s' %
key[len('x-account-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-account-meta-') and key not in (
'content-length', 'date', 'x-account-container-count',
'x-account-object-count', 'x-account-bytes-used'):
options.print_queue.put(
'%10s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
elif len(args) == 1:
try:
object_count, bytes_used = conn.head_container(args[0])
headers = conn.head_container(args[0])
object_count = int(headers.get('x-container-object-count', 0))
bytes_used = int(headers.get('x-container-bytes-used', 0))
options.print_queue.put('''
Account: %s
Container: %s
Objects: %d
Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
object_count, bytes_used))
Bytes: %d
Read ACL: %s
Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
object_count, bytes_used,
headers.get('x-container-read', ''),
headers.get('x-container-write', '')))
for key, value in headers.items():
if key.startswith('x-container-meta-'):
options.print_queue.put('%9s: %s' % ('Meta %s' %
key[len('x-container-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-container-meta-') and key not in (
'content-length', 'date', 'x-container-object-count',
'x-container-bytes-used', 'x-container-read',
'x-container-write'):
options.print_queue.put(
'%9s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Container %s not found' % repr(args[0]))
elif len(args) == 2:
try:
content_type, content_length, last_modified, etag, metadata = \
conn.head_object(args[0], args[1])
headers = conn.head_object(args[0], args[1])
options.print_queue.put('''
Account: %s
Container: %s
Object: %s
Content Type: %s
Content Length: %d
Content Length: %s
Last Modified: %s
ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
args[1], content_type, content_length,
last_modified, etag))
for key, value in metadata.items():
options.print_queue.put('%14s: %s' % ('Meta %s' % key, value))
args[1], headers.get('content-type'),
headers.get('content-length'),
headers.get('last-modified'),
headers.get('etag')))
for key, value in headers.items():
if key.startswith('x-object-meta-'):
options.print_queue.put('%14s: %s' % ('Meta %s' %
key[len('x-object-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-object-meta-') and key not in (
'content-type', 'content-length', 'last-modified',
'etag', 'date'):
options.print_queue.put(
'%14s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
@ -1092,6 +1220,63 @@ Content Length: %d
(basename(argv[0]), st_stat_help))
st_post_help = '''
post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
def st_post(options, args):
conn = Connection(options.auth, options.user, options.key)
if not args:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Account-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
try:
conn.post_account(headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
elif len(args) == 1:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Container-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
if options.read_acl is not None:
headers['X-Container-Read'] = options.read_acl
if options.write_acl is not None:
headers['X-Container-Write'] = options.write_acl
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
conn.put_container(args[0], headers=headers)
elif len(args) == 2:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Object-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
try:
conn.post_object(args[0], args[1], headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
repr('%s/%s' % (args[0], args[1])))
else:
options.error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_post_help))
st_upload_help = '''
upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
@ -1108,35 +1293,41 @@ def st_upload(options, args):
obj = path
if obj.startswith('./') or obj.startswith('.\\'):
obj = obj[2:]
metadata = {'mtime': str(getmtime(path))}
put_headers = {'x-object-meta-mtime': str(getmtime(path))}
if dir_marker:
if options.changed:
try:
ct, cl, lm, et, md = conn.head_object(args[0], obj)
headers = conn.head_object(args[0], obj)
ct = headers.get('content-type')
cl = int(headers.get('content-length'))
et = headers.get('etag')
mt = headers.get('x-object-meta-mtime')
if ct.split(';', 1)[0] == 'text/directory' and \
cl == 0 and \
et == 'd41d8cd98f00b204e9800998ecf8427e' and \
md.get('mtime') == metadata['mtime']:
mt == put_headers['x-object-meta-mtime']:
return
except ClientException, err:
if err.http_status != 404:
raise
conn.put_object(args[0], obj, '', content_length=0,
content_type='text/directory',
metadata=metadata)
headers=put_headers)
else:
if options.changed:
try:
ct, cl, lm, et, md = conn.head_object(args[0], obj)
headers = conn.head_object(args[0], obj)
cl = int(headers.get('content-length'))
mt = headers.get('x-object-meta-mtime')
if cl == getsize(path) and \
md.get('mtime') == metadata['mtime']:
mt == put_headers['x-object-meta-mtime']:
return
except ClientException, err:
if err.http_status != 404:
raise
conn.put_object(args[0], obj, open(path, 'rb'),
content_length=getsize(path),
metadata=metadata)
headers=put_headers)
if options.verbose:
options.print_queue.put(obj)
except OSError, err:
@ -1163,8 +1354,15 @@ def st_upload(options, args):
for thread in file_threads:
thread.start()
conn = create_connection()
# Try to create the container, just in case it doesn't exist. If this
# fails, it might just be because the user doesn't have container PUT
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
conn.put_container(args[0])
except:
pass
try:
for arg in args[1:]:
if isdir(arg):
_upload_dir(arg)
@ -1190,6 +1388,7 @@ Commands:
%(st_stat_help)s
%(st_list_help)s
%(st_upload_help)s
%(st_post_help)s
%(st_download_help)s
%(st_delete_help)s
@ -1215,6 +1414,17 @@ Example:
help='For the list command on containers: will roll up '
'items with the given delimiter (see Cloud Files '
'general documentation for what this means).')
parser.add_option('-r', '--read-acl', dest='read_acl',
help='Sets the Read ACL with post container commands. '
'Quick summary of ACL syntax: .r:*, .r:-.example.com, '
'.r:www.example.com, account1, account2:user2')
parser.add_option('-w', '--write-acl', dest='write_acl',
help='Sets the Write ACL with post container commands. '
'Quick summary of ACL syntax: account1, account2:user2')
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item of the syntax name:value '
'for use with post commands. This option may be '
'repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('-A', '--auth', dest='auth',
help='URL for obtaining an auth token')
parser.add_option('-U', '--user', dest='user',
@ -1235,7 +1445,7 @@ overridden with -A, -U, or -K.'''.strip('\n')
if not getattr(options, attr, None):
exit(required_help)
commands = ('delete', 'download', 'list', 'stat', 'upload')
commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
if not args or args[0] not in commands:
parser.print_usage()
if args:

59
bin/swift-auth-add-user Executable file
View File

@ -0,0 +1,59 @@
#!/usr/bin/python
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__':
default_conf = '/etc/swift/auth-server.conf'
parser = OptionParser(
usage='Usage: %prog [options] <account> <user> <password>')
parser.add_option('-c', '--conf', dest='conf', default=default_conf,
help='Configuration file to determine how to connect to the local '
'auth server (default: %s).' % default_conf)
parser.add_option('-a', '--admin', dest='admin', action='store_true',
default=False, help='Give the user administrator access; otherwise '
'the user will only have access to containers specifically allowed '
'with ACLs.')
args = argv[1:]
if not args:
args.append('-h')
(options, args) = parser.parse_args(args)
if len(args) != 3:
parser.parse_args(['-h'])
account, user, password = args
c = ConfigParser()
if not c.read(options.conf):
exit('Unable to read conf file: %s' % options.conf)
conf = dict(c.items('app:auth-server'))
host = conf.get('bind_ip', '127.0.0.1')
port = int(conf.get('bind_port', 11000))
ssl = conf.get('cert_file') is not None
path = '/account/%s/%s' % (account, user)
headers = {'X-Auth-User-Key': password}
if options.admin:
headers['X-Auth-User-Admin'] = 'true'
conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl)
resp = conn.getresponse()
if resp.status == 204:
print resp.getheader('x-storage-url')
else:
print 'Update failed: %s %s' % (resp.status, resp.reason)

47
bin/swift-auth-create-account
View File

@ -1,47 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
from sys import argv, exit
from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__':
f = '/etc/swift/auth-server.conf'
if len(argv) == 5:
f = argv[4]
elif len(argv) != 4:
exit('Syntax: %s <new_account> <new_user> <new_password> [conf_file]' %
argv[0])
new_account = argv[1]
new_user = argv[2]
new_password = argv[3]
c = ConfigParser()
if not c.read(f):
exit('Unable to read conf file: %s' % f)
conf = dict(c.items('app:auth-server'))
host = conf.get('bind_ip', '127.0.0.1')
port = int(conf.get('bind_port', 11000))
ssl = conf.get('cert_file') is not None
path = '/account/%s/%s' % (new_account, new_user)
conn = http_connect(host, port, 'PUT', path, {'x-auth-key':new_password},
ssl=ssl)
resp = conn.getresponse()
if resp.status == 204:
print resp.getheader('x-storage-url')
else:
print 'Account creation failed. (%d)' % resp.status

3
bin/swift-stats-populate
View File

@ -48,7 +48,8 @@ def put_object(connpool, container, obj, report):
global retries_done
try:
with connpool.item() as conn:
conn.put_object(container, obj, obj, metadata={'stats': obj})
conn.put_object(container, obj, obj,
headers={'x-object-meta-stats': obj})
retries_done += conn.attempts - 1
if report:
report(True)

41
bin/swift-stats-report
View File

@ -57,7 +57,8 @@ def get_error_log(prefix):
def audit(coropool, connpool, account, container_ring, object_ring, options):
begun = time()
with connpool.item() as conn:
estimated_items = [conn.head_account()[0]]
estimated_items = \
[int(conn.head_account()['x-account-container-count'])]
items_completed = [0]
retries_done = [0]
containers_missing_replicas = {}
@ -85,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
retries_done[0] += attempts - 1
found = True
if not estimated_objects:
estimated_objects = info[0]
estimated_objects = int(info['x-container-object-count'])
except ClientException, err:
if err.http_status not in (404, 507):
error_log('Giving up on /%s/%s/%s: %s' % (part, account,
@ -130,7 +131,8 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
cmarker = ''
while True:
with connpool.item() as conn:
containers = [c['name'] for c in conn.get_account(marker=cmarker)]
containers = \
[c['name'] for c in conn.get_account(marker=cmarker)[1]]
if not containers:
break
cmarker = containers[-1]
@ -142,7 +144,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
while True:
with connpool.item() as conn:
objects = [o['name'] for o in
conn.get_container(container, marker=omarker)]
conn.get_container(container, marker=omarker)[1]]
if not objects:
break
omarker = objects[-1]
@ -183,7 +185,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_dispersion_',
full_listing=True)]
full_listing=True)[1]]
containers_listed = len(containers)
if not containers_listed:
print >>stderr, 'No containers to query. Has stats-populate been run?'
@ -262,7 +264,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring, options):
with connpool.item() as conn:
try:
objects = [o['name'] for o in conn.get_container(container,
prefix='stats_object_dispersion_', full_listing=True)]
prefix='stats_object_dispersion_', full_listing=True)[1]]
except ClientException, err:
if err.http_status != 404:
raise
@ -384,7 +386,7 @@ def container_head_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def head(container):
with connpool.item() as conn:
@ -425,7 +427,7 @@ def container_get_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def get(container):
with connpool.item() as conn:
@ -463,7 +465,8 @@ def container_standard_listing_report(coropool, connpool, options):
print 'Listing big_container',
with connpool.item() as conn:
try:
value = len(conn.get_container('big_container', full_listing=True))
value = \
len(conn.get_container('big_container', full_listing=True)[1])
except ClientException, err:
if err.http_status != 404:
raise
@ -486,7 +489,7 @@ def container_prefix_listing_report(coropool, connpool, options):
try:
for x in xrange(256):
value += len(conn.get_container('big_container',
prefix=('%02x' % x), full_listing=True))
prefix=('%02x' % x), full_listing=True)[1])
except ClientException, err:
if err.http_status != 404:
raise
@ -511,7 +514,7 @@ def container_prefix_delimiter_listing_report(coropool, connpool, options):
try:
with connpool.item() as conn:
listing = conn.get_container('big_container',
marker=marker, prefix=prefix, delimiter='/')
marker=marker, prefix=prefix, delimiter='/')[1]
except ClientException, err:
if err.http_status != 404:
raise
@ -552,7 +555,7 @@ def container_delete_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def delete(container):
with connpool.item() as conn:
@ -630,7 +633,7 @@ def object_head_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def head(obj):
with connpool.item() as conn:
@ -670,7 +673,7 @@ def object_get_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def get(obj):
with connpool.item() as conn:
@ -710,7 +713,7 @@ def object_delete_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def delete(obj):
with connpool.item() as conn:
@ -791,7 +794,7 @@ Usage: %prog [options] [conf_file]
else:
options.retries = int(conf.get('retries', 5))
if not options.csv_output:
csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
coropool = GreenPool(size=concurrency)
@ -908,8 +911,8 @@ Usage: %prog [options] [conf_file]
if options.csv_output != 'None':
try:
if not os.path.exists(csv_output):
f = open(csv_output, 'wb')
if not os.path.exists(options.csv_output):
f = open(options.csv_output, 'wb')
f.write('Timestamp,'
'Container Dispersion Report Time,'
'Container Dispersion Report Value,'
@ -936,7 +939,7 @@ Usage: %prog [options] [conf_file]
'Object DELETE Report Success Rate\r\n')
csv = csv.writer(f)
else:
csv = csv.writer(open(csv_output, 'ab'))
csv = csv.writer(open(options.csv_output, 'ab'))
csv.writerow(report)
except Exception, err:
print >>stderr, 'Could not write CSV report:', err

2
doc/source/auth.rst
View File

@ -4,7 +4,7 @@
Developer's Authorization
*************************
.. _auth-server:
.. _auth_server:
Auth Server
===========

484
doc/source/development_auth.rst Normal file
View File

@ -0,0 +1,484 @@
==========================
Auth Server and Middleware
==========================
--------------------------------------------
Creating Your Own Auth Server and Middleware
--------------------------------------------
The included swift/common/middleware/auth.py is a good minimal example of how
to create auth middleware. The main points are that the auth middleware can
reject requests up front, before they ever get to the Swift Proxy application,
and afterwards when the proxy issues callbacks to verify authorization.
It's generally good to separate the authentication and authorization
procedures. Authentication verifies that a request actually comes from who it
says it does. Authorization verifies the 'who' has access to the resource(s)
the request wants.
Authentication is performed on the request before it ever gets to the Swift
Proxy application. The identity information is gleaned from the request,
validated in some way, and the validation information is added to the WSGI
environment as needed by the future authorization procedure. What exactly is
added to the WSGI environment is solely dependent on what the installed
authorization procedures need; the Swift Proxy application itself needs no
specific information, it just passes it along. Convention has
environ['REMOTE_USER'] set to the authenticated user string but often more
information is needed than just that.
The included DevAuth will set the REMOTE_USER to a comma separated list of
groups the user belongs to. The first group will be the "user's group", a group
that only the user belongs to. The second group will be the "account's group",
a group that includes all users for that auth account (different than the
storage account). The third group is optional and is the storage account
string. If the user does not have admin access to the account, the third group
will be omitted.
It is highly recommended that authentication server implementers prefix their
tokens and Swift storage accounts they create with a configurable reseller
prefix (`AUTH_` by default with the included DevAuth). This prefix will allow
deconflicting with other authentication servers that might be using the same
Swift cluster. Otherwise, the Swift cluster will have to try all the resellers
until one validates a token or all fail.
A restriction with group names is that no group name should begin with a period
'.' as that is reserved for internal Swift use (such as the .r for referrer
designations as you'll see later).
Example Authentication with DevAuth:
* Token AUTH_tkabcd is given to the DevAuth middleware in a request's
X-Auth-Token header.
* The DevAuth middleware makes a validate token AUTH_tkabcd call to the
external DevAuth server.
* The external DevAuth server validates the token AUTH_tkabcd and discovers
it matches the "tester" user within the "test" account for the storage
account "AUTH_storage_xyz".
* The external DevAuth server responds with "X-Auth-Groups:
test:tester,test,AUTH_storage_xyz"
* Now this user will have full access (via authorization procedures later)
to the AUTH_storage_xyz Swift storage account and access to other storage
accounts with the same `AUTH_` reseller prefix and has an ACL specifying
at least one of those three groups returned.
Authorization is performed through callbacks by the Swift Proxy server to the
WSGI environment's swift.authorize value, if one is set. The swift.authorize
value should simply be a function that takes a webob.Request as an argument and
returns None if access is granted or returns a callable(environ,
start_response) if access is denied. This callable is a standard WSGI callable.
Generally, you should return 403 Forbidden for requests by an authenticated
user and 401 Unauthorized for an unauthenticated request. For example, here's
an authorize function that only allows GETs (in this case you'd probably return
405 Method Not Allowed, but ignore that for the moment).::
from webob import HTTPForbidden, HTTPUnauthorized
def authorize(req):
if req.method == 'GET':
return None
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
Adding the swift.authorize callback is often done by the authentication
middleware as authentication and authorization are often paired together. But,
you could create separate authorization middleware that simply sets the
callback before passing on the request. To continue our example above::
from webob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, environ, start_response):
environ['swift.authorize'] = self.authorize
return self.app(environ, start_response)
def authorize(self, req):
if req.method == 'GET':
return None
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return Authorization(app, conf)
return auth_filter
The Swift Proxy server will call swift.authorize after some initial work, but
before truly trying to process the request. Positive authorization at this
point will cause the request to be fully processed immediately. A denial at
this point will immediately send the denial response for most operations.
But for some operations that might be approved with more information, the
additional information will be gathered and added to the WSGI environment and
then swift.authorize will be called once more. These are called delay_denial
requests and currently include container read requests and object read and
write requests. For these requests, the read or write access control string
(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl'
attribute in the webob.Request passed to swift.authorize.
The delay_denial procedures allow skipping possibly expensive access control
string retrievals for requests that can be approved without that information,
such as administrator or account owner requests.
To further our example, we now will approve all requests that have the access
control string set to same value as the authenticated user string. Note that
you probably wouldn't do this exactly as the access control string represents a
list rather than a single user, but it'll suffice for this example::
from webob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, environ, start_response):
environ['swift.authorize'] = self.authorize
return self.app(environ, start_response)
def authorize(self, req):
# Allow anyone to perform GET requests
if req.method == 'GET':
return None
# Allow any request where the acl equals the authenticated user
if getattr(req, 'acl', None) == req.remote_user:
return None
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return Authorization(app, conf)
return auth_filter
The access control string has a standard format included with Swift, though
this can be overridden if desired. The standard format can be parsed with
swift.common.middleware.acl.parse_acl which converts the string into two arrays
of strings: (referrers, groups). The referrers allow comparing the request's
Referer header to control access. The groups allow comparing the
request.remote_user (or other sources of group information) to control access.
Checking referrer access can be accomplished by using the
swift.common.middleware.acl.referrer_allowed function. Checking group access is
usually a simple string comparison.
Let's continue our example to use parse_acl and referrer_allowed. Now we'll
only allow GETs after a referrer check and any requests after a group check::
from swift.common.middleware.acl import parse_acl, referrer_allowed
from webob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, environ, start_response):
environ['swift.authorize'] = self.authorize
return self.app(environ, start_response)
def authorize(self, req):
if hasattr(req, 'acl'):
referrers, groups = parse_acl(req.acl)
if req.method == 'GET' and referrer_allowed(req, referrers):
return None
if req.remote_user and groups and req.remote_user in groups:
return None
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return Authorization(app, conf)
return auth_filter
The access control strings are set with PUTs and POSTs to containers with the
X-Container-Read and X-Container-Write headers. Swift allows these strings to
be set to any value, though it's very useful to validate the strings meet the
desired format and return a useful error to the user if they don't.
To support this validation, the Swift Proxy application will call the WSGI
environment's swift.clean_acl callback whenever one of these headers is to be
written. The callback should take a header name and value as its arguments. It
should return the cleaned value to save if valid or raise a ValueError with a
reasonable error message if not.
There is an included swift.common.middleware.acl.clean_acl that validates the
standard Swift format. Let's improve our example by making use of that::
from swift.common.middleware.acl import \
clean_acl, parse_acl, referrer_allowed
from webob import HTTPForbidden, HTTPUnauthorized
class Authorization(object):
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, environ, start_response):
environ['swift.authorize'] = self.authorize
environ['swift.clean_acl'] = clean_acl
return self.app(environ, start_response)
def authorize(self, req):
if hasattr(req, 'acl'):
referrers, groups = parse_acl(req.acl)
if req.method == 'GET' and referrer_allowed(req, referrers):
return None
if req.remote_user and groups and req.remote_user in groups:
return None
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return Authorization(app, conf)
return auth_filter
Now, if you want to override the format for access control strings you'll have
to provide your own clean_acl function and you'll have to do your own parsing
and authorization checking for that format. It's highly recommended you use the
standard format simply to support the widest range of external tools, but
sometimes that's less important than meeting certain ACL requirements.
----------------------------
Integrating With repoze.what
----------------------------
Here's an example of integration with repoze.what, though honestly it just does
what the default swift/common/middleware/auth.py does in a slightly different
way. I'm no repoze.what expert by any stretch; this is just included here to
hopefully give folks a start on their own code if they want to use
repoze.what::
from time import time
from eventlet.timeout import Timeout
from repoze.what.adapters import BaseSourceAdapter
from repoze.what.middleware import setup_auth
from repoze.what.predicates import in_any_group, NotAuthorizedError
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, split_path
from webob.exc import HTTPForbidden, HTTPUnauthorized
class DevAuthorization(object):
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, environ, start_response):
environ['swift.authorize'] = self.authorize
environ['swift.clean_acl'] = clean_acl
return self.app(environ, start_response)
def authorize(self, req):
version, account, container, obj = split_path(req.path, 1, 4, True)
if not account:
return self.denied_response(req)
referrers, groups = parse_acl(getattr(req, 'acl', None))
if referrer_allowed(req, referrers):
return None
try:
in_any_group(account, *groups).check_authorization(req.environ)
except NotAuthorizedError:
return self.denied_response(req)
return None
def denied_response(self, req):
if req.remote_user:
return HTTPForbidden(request=req)
else:
return HTTPUnauthorized(request=req)
class DevIdentifier(object):
def __init__(self, conf):
self.conf = conf
def identify(self, env):
return {'token':
env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))}
def remember(self, env, identity):
return []
def forget(self, env, identity):
return []
class DevAuthenticator(object):
def __init__(self, conf):
self.conf = conf
self.auth_host = conf.get('ip', '127.0.0.1')
self.auth_port = int(conf.get('port', 11000))
self.ssl = \
conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes')
self.timeout = int(conf.get('node_timeout', 10))
def authenticate(self, env, identity):
token = identity.get('token')
if not token:
return None
memcache_client = cache_from_env(env)
key = 'devauth/%s' % token
cached_auth_data = memcache_client.get(key)
if cached_auth_data:
start, expiration, user = cached_auth_data
if time() - start <= expiration:
return user
with Timeout(self.timeout):
conn = http_connect(self.auth_host, self.auth_port, 'GET',
'/token/%s' % token, ssl=self.ssl)
resp = conn.getresponse()
resp.read()
conn.close()
if resp.status == 204:
expiration = float(resp.getheader('x-auth-ttl'))
user = resp.getheader('x-auth-user')
memcache_client.set(key, (time(), expiration, user),
timeout=expiration)
return user
return None
class DevChallenger(object):
def __init__(self, conf):
self.conf = conf
def challenge(self, env, status, app_headers, forget_headers):
def no_challenge(env, start_response):
start_response(str(status), [])
return []
return no_challenge
class DevGroupSourceAdapter(BaseSourceAdapter):
def __init__(self, *args, **kwargs):
super(DevGroupSourceAdapter, self).__init__(*args, **kwargs)
self.sections = {}
def _get_all_sections(self):
return self.sections
def _get_section_items(self, section):
return self.sections[section]
def _find_sections(self, credentials):
return credentials['repoze.what.userid'].split(',')
def _include_items(self, section, items):
self.sections[section] |= items
def _exclude_items(self, section, items):
for item in items:
self.sections[section].remove(item)
def _item_is_included(self, section, item):
return item in self.sections[section]
def _create_section(self, section):
self.sections[section] = set()
def _edit_section(self, section, new_section):
self.sections[new_section] = self.sections[section]
del self.sections[section]
def _delete_section(self, section):
del self.sections[section]
def _section_exists(self, section):
return self.sections.has_key(section)
class DevPermissionSourceAdapter(BaseSourceAdapter):
def __init__(self, *args, **kwargs):
super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs)
self.sections = {}
def _get_all_sections(self):
return self.sections
def _get_section_items(self, section):
return self.sections[section]
def _find_sections(self, group_name):
return set([n for (n, p) in self.sections.items()
if group_name in p])
def _include_items(self, section, items):
self.sections[section] |= items
def _exclude_items(self, section, items):
for item in items:
self.sections[section].remove(item)
def _item_is_included(self, section, item):
return item in self.sections[section]
def _create_section(self, section):
self.sections[section] = set()
def _edit_section(self, section, new_section):
self.sections[new_section] = self.sections[section]
del self.sections[section]
def _delete_section(self, section):
del self.sections[section]
def _section_exists(self, section):
return self.sections.has_key(section)
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return setup_auth(DevAuthorization(app, conf),
group_adapters={'all_groups': DevGroupSourceAdapter()},
permission_adapters={'all_perms': DevPermissionSourceAdapter()},
identifiers=[('devauth', DevIdentifier(conf))],
authenticators=[('devauth', DevAuthenticator(conf))],
challengers=[('devauth', DevChallenger(conf))])
return auth_filter

24
doc/source/development_saio.rst
View File

@ -526,25 +526,17 @@ good idea what to do on other environments.
#. `remakerings`
#. `cd ~/swift/trunk; ./.unittests`
#. `startmain` (The ``Unable to increase file descriptor limit. Running as non-root?`` warnings are expected and ok.)
#. `swift-auth-create-account test tester testing`
#. `swift-auth-add-user --admin test tester testing`
#. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0``
#. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
#. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat`
#. Create `/etc/swift/func_test.conf`::
auth_host = 127.0.0.1
auth_port = 11000
auth_ssl = no
account = test
username = tester
password = testing
collate = C
#. `cd ~/swift/trunk; ./.functests`
#. `cd ~/swift/trunk; ./.probetests` (Note for future reference: probe tests
will reset your environment)
#. `swift-auth-add-user --admin test2 tester2 testing2`
#. `swift-auth-add-user test tester3 testing3`
#. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf`
#. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete
everything in the configured accounts.)
#. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your
environment as they call `resetswift` for each test.)
If you plan to work on documentation (and who doesn't?!):

4
doc/source/howto_cyberduck.rst
View File

@ -107,9 +107,9 @@ Installing Swift For Use With Cyberduck
cert_file = /etc/swift/cert.crt
key_file = /etc/swift/cert.key
#. Use swift-auth-create-account to create a new account::
#. Use swift-auth-add-user to create a new account and admin user::
ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-create-account a3 b3 c3
ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-add-user --admin a3 b3 c3
https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1/06228ccf-6d0a-4395-889e-e971e8de8781
.. note::

1
doc/source/index.rst
View File

@ -32,6 +32,7 @@ Development:
development_guidelines
development_saio
development_auth
Deployment:

9
doc/source/misc.rst
View File

@ -42,6 +42,15 @@ Auth
:members:
:show-inheritance:
.. _acls:
ACLs
====
.. automodule:: swift.common.middleware.acl
:members:
:show-inheritance:
.. _wsgi:
WSGI

27
doc/source/overview_auth.rst
View File

@ -6,9 +6,9 @@ The Auth System
Developer Auth
--------------
The auth system for Swift is based on the auth system from the existing
Rackspace architecture -- actually from a few existing auth systems --
and is therefore a bit disjointed. The distilled points about it are:
The auth system for Swift is loosely based on the auth system from the existing
Rackspace architecture -- actually from a few existing auth systems -- and is
therefore a bit disjointed. The distilled points about it are:
* The authentication/authorization part is outside Swift itself
* The user of Swift passes in an auth token with each request
@ -23,13 +23,16 @@ of something unique, some use "something else" but the salient point is that
the token is a string which can be sent as-is back to the auth system for
validation.
An auth call is given the auth token and the Swift account hash. For a valid
token, the auth system responds with a session TTL and overall expiration in
seconds from now. Swift does not honor the session TTL but will cache the
token up to the expiration time. Tokens can be purged through a call to the
auth system.
Swift will make calls to the external auth system, giving the auth token to be
validated. For a valid token, the auth system responds with an overall
expiration in seconds from now. Swift will cache the token up to the expiration
time. The included devauth also has the concept of admin and non-admin users
within an account. Admin users can do anything within the account. Non-admin
users can only perform operations per container based on the container's
X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see
:mod:`swift.common.middleware.acl`
The user starts a session by sending a ReST request to that auth system
The user starts a session by sending a ReST request to the external auth system
to receive the auth token and a URL to the Swift system.
--------------
@ -40,8 +43,10 @@ Auth is written as wsgi middleware, so implementing your own auth is as easy
as writing new wsgi middleware, and plugging it in to the proxy server.
The current middleware is implemented in the DevAuthMiddleware class in
swift/common/auth.py, and should be a good starting place for implemeting
your own auth.
swift/common/middleware/auth.py, and should be a good starting place for
implementing your own auth.
Also, see :doc:`development_auth`.
------------------
History and Future

6
etc/proxy-server.conf-sample
View File

@ -37,6 +37,12 @@ use = egg:swift#proxy
[filter:auth]
use = egg:swift#auth
# The reseller prefix will verify a token begins with this prefix before even
# attempting to validate it with the external authentication server. Also, with
# authorization, only Swift storage accounts with this prefix will be
# authorized by this middleware. Useful if multiple auth systems are in use for
# one Swift cluster.
# reseller_prefix = AUTH
# ip = 127.0.0.1
# port = 11000
# ssl = false

2
setup.py
View File

@ -61,7 +61,7 @@ setup(
'bin/st', 'bin/swift-account-auditor',
'bin/swift-account-audit', 'bin/swift-account-reaper',
'bin/swift-account-replicator', 'bin/swift-account-server',
'bin/swift-auth-create-account',
'bin/swift-auth-add-user',
'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
'bin/swift-container-auditor',
'bin/swift-container-replicator',

2
swift/account/reaper.py
View File

@ -310,7 +310,7 @@ class AccountReaper(Daemon):
try:
objects = direct_get_container(node, part, account, container,
marker=marker, conn_timeout=self.conn_timeout,
response_timeout=self.node_timeout)
response_timeout=self.node_timeout)[1]
self.stats_return_codes[2] = \
self.stats_return_codes.get(2, 0) + 1
except ClientException, err:

222
swift/auth/server.py
View File

@ -22,6 +22,7 @@ from time import gmtime, strftime, time
from urllib import unquote, quote
from uuid import uuid4
import sqlite3
from webob import Request, Response
from webob.exc import HTTPBadRequest, HTTPNoContent, HTTPUnauthorized, \
HTTPServiceUnavailable, HTTPNotFound
@ -58,17 +59,18 @@ class AuthController(object):
* The user makes a ReST call to the Swift cluster using the url given with
the token as the X-Auth-Token header.
* The Swift cluster makes an ReST call to the auth server to validate the
token for the given account hash, caching the result for future requests
up to the expiration the auth server returns.
* The auth server validates the token / account hash given and returns the
expiration for the token.
token, caching the result for future requests up to the expiration the
auth server returns.
* The auth server validates the token given and returns the expiration for
the token.
* The Swift cluster completes the user's request.
Another use case is creating a new account:
Another use case is creating a new user:
* The developer makes a ReST call to create a new account.
* The auth server makes ReST calls to the Swift cluster's account servers
to create a new account on its end.
* The developer makes a ReST call to create a new user.
* If the account for the user does not yet exist, the auth server makes
ReST calls to the Swift cluster's account servers to create a new account
on its end.
* The auth server records the information in its database.
A last use case is recreating existing accounts; this is really only useful
@ -92,6 +94,9 @@ class AuthController(object):
def __init__(self, conf, ring=None):
self.logger = get_logger(conf)
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if self.reseller_prefix and self.reseller_prefix[-1] != '_':
self.reseller_prefix += '_'
self.default_cluster_url = \
conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1')
self.token_life = int(conf.get('token_life', 86400))
@ -103,17 +108,33 @@ class AuthController(object):
Ring(os.path.join(self.swift_dir, 'account.ring.gz'))
self.db_file = os.path.join(self.swift_dir, 'auth.db')
self.conn = get_db_connection(self.db_file, okay_to_create=True)
try:
self.conn.execute('SELECT admin FROM account LIMIT 1')
except sqlite3.OperationalError, err:
if str(err) == 'no such column: admin':
self.conn.execute("ALTER TABLE account ADD COLUMN admin TEXT")
self.conn.execute("UPDATE account SET admin = 't'")
self.conn.execute('''CREATE TABLE IF NOT EXISTS account (
account TEXT, url TEXT, cfaccount TEXT,
user TEXT, password TEXT)''')
user TEXT, password TEXT, admin TEXT)''')
self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account
ON account (account)''')
try:
self.conn.execute('SELECT user FROM token LIMIT 1')
except sqlite3.OperationalError, err:
if str(err) == 'no such column: user':
self.conn.execute('DROP INDEX IF EXISTS ix_token_created')
self.conn.execute('DROP INDEX IF EXISTS ix_token_cfaccount')
self.conn.execute('DROP TABLE IF EXISTS token')
self.conn.execute('''CREATE TABLE IF NOT EXISTS token (
cfaccount TEXT, token TEXT, created FLOAT)''')
self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount
ON token (cfaccount)''')
token TEXT, created FLOAT,
account TEXT, user TEXT, cfaccount TEXT)''')
self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token
ON token (token)''')
self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created
ON token (created)''')
self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account
ON token (account)''')
self.conn.commit()
def add_storage_account(self, account_name=''):
@ -129,7 +150,7 @@ class AuthController(object):
begin = time()
orig_account_name = account_name
if not account_name:
account_name = str(uuid4())
account_name = '%s%s' % (self.reseller_prefix, uuid4().hex)
partition, nodes = self.account_ring.get_nodes(account_name)
headers = {'X-Timestamp': normalize_timestamp(time()),
'x-cf-trans-id': 'tx' + str(uuid4())}
@ -202,77 +223,99 @@ class AuthController(object):
(time() - self.token_life,))
conn.commit()
def validate_token(self, token, account_hash):
def validate_token(self, token):
"""
Tests if the given token is a valid token
:param token: The token to validate
:param account_hash: The account hash the token is being used with
:returns: TTL if valid, False otherwise
:returns: (TTL, account, user, cfaccount) if valid, False otherwise.
cfaccount will be None for users without admin access.
"""
begin = time()
self.purge_old_tokens()
rv = False
with self.get_conn() as conn:
row = conn.execute('''
SELECT created FROM token
WHERE cfaccount = ? AND token = ?''',
(account_hash, token)).fetchone()
SELECT created, account, user, cfaccount FROM token
WHERE token = ?''',
(token,)).fetchone()
if row is not None:
created = row[0]
if time() - created >= self.token_life:
conn.execute('''
DELETE FROM token
WHERE cfaccount = ? AND token = ?''',
(account_hash, token))
DELETE FROM token WHERE token = ?''', (token,))
conn.commit()
else:
rv = self.token_life - (time() - created)
self.logger.info('validate_token(%s, %s, _, _) = %s [%.02f]' %
(repr(token), repr(account_hash), repr(rv),
time() - begin))
rv = (self.token_life - (time() - created), row[1], row[2],
row[3])
self.logger.info('validate_token(%s, _, _) = %s [%.02f]' %
(repr(token), repr(rv), time() - begin))
return rv
def create_account(self, new_account, new_user, new_password):
def create_user(self, account, user, password, admin=False):
"""
Handles the create_account call for developers, used to request
an account be created both on a Swift cluster and in the auth server
database.
Handles the create_user call for developers, used to request a user be
added in the auth server database. If the account does not yet exist,
it will be created on the Swift cluster and the details recorded in the
auth server database.
This will make ReST requests to the Swift cluster's account servers
to have an account created on its side. The resulting account hash
along with the URL to use to access the account, the account name, the
user name, and the password is recorded in the auth server's database.
The url is constructed now and stored separately to support changing
the configuration file's default_cluster_url for directing new accounts
to a different Swift cluster while still supporting old accounts going
to the Swift clusters they were created on.
The url for the storage account is constructed now and stored
separately to support changing the configuration file's
default_cluster_url for directing new accounts to a different Swift
cluster while still supporting old accounts going to the Swift clusters
they were created on.
:param new_account: The name for the new account
:param new_user: The name for the new user
:param new_password: The password for the new account
Currently, updating a user's information (password, admin access) must
be done by directly updating the sqlite database.
:returns: False if the create fails, storage url if successful
:param account: The name for the new account
:param user: The name for the new user
:param password: The password for the new account
:param admin: If true, the user will be granted full access to the
account; otherwise, another user will have to add the
user to the ACLs for containers to grant access.
:returns: False if the create fails, 'already exists' if the user
already exists, or storage url if successful
"""
begin = time()
if not all((new_account, new_user, new_password)):
if not all((account, user, password)):
return False
account_hash = self.add_storage_account()
if not account_hash:
self.logger.info(
'FAILED create_account(%s, %s, _,) [%.02f]' %
(repr(new_account), repr(new_user), time() - begin))
return False
url = self.default_cluster_url.rstrip('/') + '/' + account_hash
with self.get_conn() as conn:
row = conn.execute(
'SELECT url FROM account WHERE account = ? AND user = ?',
(account, user)).fetchone()
if row:
self.logger.info(
'ALREADY EXISTS create_user(%s, %s, _, %s) [%.02f]' %
(repr(account), repr(user), repr(admin),
time() - begin))
return 'already exists'
row = conn.execute(
'SELECT url, cfaccount FROM account WHERE account = ?',
(account,)).fetchone()
if row:
url = row[0]
account_hash = row[1]
else:
account_hash = self.add_storage_account()
if not account_hash:
self.logger.info(
'FAILED create_user(%s, %s, _, %s) [%.02f]' %
(repr(account), repr(user), repr(admin),
time() - begin))
return False
url = self.default_cluster_url.rstrip('/') + '/' + account_hash
conn.execute('''INSERT INTO account
(account, url, cfaccount, user, password)
VALUES (?, ?, ?, ?, ?)''',
(new_account, url, account_hash, new_user, new_password))
(account, url, cfaccount, user, password, admin)
VALUES (?, ?, ?, ?, ?, ?)''',
(account, url, account_hash, user, password,
admin and 't' or ''))
conn.commit()
self.logger.info(
'SUCCESS create_account(%s, %s, _) = %s [%.02f]' %
(repr(new_account), repr(new_user), repr(url), time() - begin))
'SUCCESS create_user(%s, %s, _, %s) = %s [%.02f]' %
(repr(account), repr(user), repr(admin), repr(url),
time() - begin))
return url
def recreate_accounts(self):
@ -285,8 +328,8 @@ class AuthController(object):
"""
begin = time()
with self.get_conn() as conn:
account_hashes = [r[0] for r in
conn.execute('SELECT cfaccount FROM account').fetchall()]
account_hashes = [r[0] for r in conn.execute(
'SELECT distinct(cfaccount) FROM account').fetchall()]
failures = []
for i, account_hash in enumerate(account_hashes):
if not self.add_storage_account(account_hash):
@ -298,36 +341,47 @@ class AuthController(object):
def handle_token(self, request):
"""
Hanles ReST request from Swift to validate tokens
Handles ReST requests from Swift to validate tokens
Valid URL paths:
* GET /token/<account-hash>/<token>
* GET /token/<token>
If the HTTP equest returns with a 204, then the token is valid,
and the TTL of the token will be available in the X-Auth-Ttl header.
If the HTTP request returns with a 204, then the token is valid, the
TTL of the token will be available in the X-Auth-Ttl header, and a
comma separated list of the "groups" the user belongs to will be in the
X-Auth-Groups header.
:param request: webob.Request object
"""
try:
_, account_hash, token = split_path(request.path, minsegs=3)
_, token = split_path(request.path, minsegs=2)
except ValueError:
return HTTPBadRequest()
ttl = self.validate_token(token, account_hash)
if not ttl:
# Retrieves (TTL, account, user, cfaccount) if valid, False otherwise
validation = self.validate_token(token)
if not validation:
return HTTPNotFound()
return HTTPNoContent(headers={'x-auth-ttl': ttl})
groups = ['%s:%s' % (validation[1], validation[2]), validation[1]]
if validation[3]: # admin access to a cfaccount
groups.append(validation[3])
return HTTPNoContent(headers={'X-Auth-TTL': validation[0],
'X-Auth-Groups': ','.join(groups)})
def handle_account_create(self, request):
def handle_add_user(self, request):
"""
Handles Rest requests from developers to have an account created.
Handles Rest requests from developers to have a user added. If the
account specified doesn't exist, it will also be added. Currently,
updating a user's information (password, admin access) must be done by
directly updating the sqlite database.
Valid URL paths:
* PUT /account/<account-name>/<user-name> - create the account
Valid headers:
* X-Auth-Key: <password> (Only required when creating an account)
* X-Auth-User-Key: <password>
* X-Auth-User-Admin: <true|false>
If the HTTP request returns with a 204, then the account was created,
If the HTTP request returns with a 204, then the user was added,
and the storage url will be available in the X-Storage-Url header.
:param request: webob.Request object
@ -336,10 +390,13 @@ class AuthController(object):
_, account_name, user_name = split_path(request.path, minsegs=3)
except ValueError:
return HTTPBadRequest()
if 'X-Auth-Key' not in request.headers:
return HTTPBadRequest('X-Auth-Key is required')
password = request.headers['x-auth-key']
storage_url = self.create_account(account_name, user_name, password)
if 'X-Auth-User-Key' not in request.headers:
return HTTPBadRequest('X-Auth-User-Key is required')
password = request.headers['x-auth-user-key']
storage_url = self.create_user(account_name, user_name, password,
request.headers.get('x-auth-user-admin') == 'true')
if storage_url == 'already exists':
return HTTPBadRequest(storage_url)
if not storage_url:
return HTTPServiceUnavailable()
return HTTPNoContent(headers={'x-storage-url': storage_url})
@ -414,23 +471,26 @@ class AuthController(object):
self.purge_old_tokens()
with self.get_conn() as conn:
row = conn.execute('''
SELECT cfaccount, url FROM account
SELECT cfaccount, url, admin FROM account
WHERE account = ? AND user = ? AND password = ?''',
(account, user, password)).fetchone()
if row is None:
return HTTPUnauthorized()
cfaccount = row[0]
url = row[1]
row = conn.execute('SELECT token FROM token WHERE cfaccount = ?',
(cfaccount,)).fetchone()
admin = row[2] == 't'
row = conn.execute('''
SELECT token FROM token WHERE account = ? AND user = ?''',
(account, user)).fetchone()
if row:
token = row[0]
else:
token = 'tk' + str(uuid4())
token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
conn.execute('''
INSERT INTO token (cfaccount, token, created)
VALUES (?, ?, ?)''',
(cfaccount, token, time()))
INSERT INTO token
(token, created, account, user, cfaccount)
VALUES (?, ?, ?, ?, ?)''',
(token, time(), account, user, admin and cfaccount or ''))
conn.commit()
return HTTPNoContent(headers={'x-auth-token': token,
'x-storage-token': token,
@ -457,7 +517,7 @@ class AuthController(object):
elif req.method == 'GET' and req.path.startswith('/token/'):
handler = self.handle_token
elif req.method == 'PUT' and req.path.startswith('/account/'):
handler = self.handle_account_create
handler = self.handle_add_user
elif req.method == 'POST' and \
req.path == '/recreate_accounts':
handler = self.handle_account_recreate

215
swift/common/client.py
View File

@ -194,18 +194,21 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of accounts
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_account(url, token, marker, limit, prefix, http_conn)
rv = get_account(url, token, marker, limit, prefix, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
marker = listing[-1]['name']
listing = get_account(url, token, marker, limit, prefix, http_conn)
listing = \
get_account(url, token, marker, limit, prefix, http_conn)[1]
if listing:
rv.extend(listing)
return rv
parsed, conn = http_conn
qs = 'format=json'
@ -218,6 +221,9 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
conn.request('GET', '%s?%s' % (parsed.path, qs), '',
{'X-Auth-Token': token})
resp = conn.getresponse()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@ -226,8 +232,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
http_reason=resp.reason)
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_account(url, token, http_conn=None):
@ -238,7 +244,8 @@ def head_account(url, token, http_conn=None):
:param token: auth token
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (container count, object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -253,9 +260,36 @@ def head_account(url, token, http_conn=None):
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-account-container-count', 0)), \
int(resp.getheader('x-account-object-count', 0)), \
int(resp.getheader('x-account-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def post_account(url, token, headers, http_conn=None):
"""
Update an account's metadata.
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
headers['X-Auth-Token'] = token
conn.request('POST', parsed.path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def get_container(url, token, container, marker=None, limit=None,
@ -275,23 +309,25 @@ def get_container(url, token, container, marker=None, limit=None,
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, http_conn)
prefix, delimiter, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
path = '%s/%s' % (parsed.path, quote(container))
@ -312,10 +348,13 @@ def get_container(url, token, container, marker=None, limit=None,
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_query=qs,
http_status=resp.status, http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None):
@ -327,7 +366,8 @@ def head_container(url, token, container, http_conn=None):
:param container: container name to get stats for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -343,17 +383,20 @@ def head_container(url, token, container, http_conn=None):
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count', 0)), \
int(resp.getheader('x-container-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_container(url, token, container, http_conn=None):
def put_container(url, token, container, headers=None, http_conn=None):
"""
Create a container
:param url: storage URL
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP PUT request failed
@ -363,7 +406,10 @@ def put_container(url, token, container, http_conn=None):
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('PUT', path, '', {'X-Auth-Token': token})
if not headers:
headers = {}
headers['X-Auth-Token'] = token
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
@ -373,6 +419,34 @@ def put_container(url, token, container, http_conn=None):
http_reason=resp.reason)
def post_container(url, token, container, headers, http_conn=None):
"""
Update a container's metadata.
:param url: storage URL
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def delete_container(url, token, container, http_conn=None):
"""
Delete a container
@ -410,8 +484,12 @@ def get_object(url, token, container, name, http_conn=None,
:param name: object name to get
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param resp_chunk_size: if defined, chunk size of data to read
:returns: a list of objects
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
request.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if http_conn:
@ -426,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None,
raise ClientException('Object GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
if resp_chunk_size:
def _object_body():
@ -440,12 +514,10 @@ def get_object(url, token, container, name, http_conn=None,
object_body = _object_body()
else:
object_body = resp.read()
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata, \
object_body
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def head_object(url, token, container, name, http_conn=None):
@ -458,8 +530,8 @@ def head_object(url, token, container, name, http_conn=None):
:param name: object name to get info for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (content type, content length, last modfied, etag,
dictionary of metadata)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -474,20 +546,15 @@ def head_object(url, token, container, name, http_conn=None):
raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port, http_path=path,
http_status=resp.status, http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_object(url, token, container, name, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None, http_conn=None):
def put_object(url, token, container, name, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None, headers=None,
http_conn=None):
"""
Put an object
@ -495,12 +562,12 @@ def put_object(url, token, container, name, contents, metadata={},
:param token: auth token
:param container: container name that the object is in
:param name: object name to put
:param contents: file like object to read object data from
:param metadata: dictionary of object metadata
:param contents: a string or a file like object to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param chunk_size: chunk size of data to write
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: etag from server response
@ -511,9 +578,9 @@ def put_object(url, token, container, name, contents, metadata={},
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
if not headers:
headers = {}
headers['X-Auth-Token'] = token
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
@ -549,15 +616,15 @@ def put_object(url, token, container, name, contents, metadata={},
return resp.getheader('etag').strip('"')
def post_object(url, token, container, name, metadata, http_conn=None):
def post_object(url, token, container, name, headers, http_conn=None):
"""
Change object metadata
Update object metadata
:param url: storage URL
:param token: auth token
:param container: container name that the object is in
:param name: object name to change
:param metadata: dictionary of object metadata
:param name: name of the object to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
@ -567,9 +634,7 @@ def post_object(url, token, container, name, metadata, http_conn=None):
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
@ -682,6 +747,10 @@ class Connection(object):
return self._retry(get_account, marker=marker, limit=limit,
prefix=prefix, full_listing=full_listing)
def post_account(self, headers):
"""Wrapper for :func:`post_account`"""
return self._retry(post_account, headers)
def head_container(self, container):
"""Wrapper for :func:`head_container`"""
return self._retry(head_container, container)
@ -696,9 +765,13 @@ class Connection(object):
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
def put_container(self, container):
def put_container(self, container, headers=None):
"""Wrapper for :func:`put_container`"""
return self._retry(put_container, container)
return self._retry(put_container, container, headers=headers)
def post_container(self, container, headers):
"""Wrapper for :func:`post_container`"""
return self._retry(post_container, container, headers)
def delete_container(self, container):
"""Wrapper for :func:`delete_container`"""
@ -713,17 +786,17 @@ class Connection(object):
return self._retry(get_object, container, obj,
resp_chunk_size=resp_chunk_size)
def put_object(self, container, obj, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None):
def put_object(self, container, obj, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None,
headers=None):
"""Wrapper for :func:`put_object`"""
return self._retry(put_object, container, obj, contents,
metadata=metadata, content_length=content_length, etag=etag,
chunk_size=chunk_size, content_type=content_type)
content_length=content_length, etag=etag, chunk_size=chunk_size,
content_type=content_type, headers=headers)
def post_object(self, container, obj, metadata):
def post_object(self, container, obj, headers):
"""Wrapper for :func:`post_object`"""
return self._retry(post_object, container, obj, metadata)
return self._retry(post_object, container, obj, headers)
def delete_object(self, container, obj):
"""Wrapper for :func:`delete_object`"""

71
swift/common/direct_client.py
View File

@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
:param container: container name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s' % (account, container)
with Timeout(conn_timeout):
@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count')), \
int(resp.getheader('x-container-bytes-used'))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def direct_get_container(node, part, account, container, marker=None,
@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None,
:param delimeter: delimeter for the query
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
"""
path = '/%s/%s' % (account, container)
qs = 'format=json'
@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def direct_delete_container(node, part, account, container, conn_timeout=5,
@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
'DELETE', path, headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException(
'Container server %s:%s direct DELETE %s gave status %s' %
@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return resp
def direct_head_object(node, part, account, container, obj, conn_timeout=5,
@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: tuple of (content-type, object size, last modified timestamp,
etag, metadata dictionary)
:returns: a dict containing the response's headers (all header names will
be lowercase)
"""
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length')), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def direct_get_object(node, part, account, container, obj, conn_timeout=5,
response_timeout=15):
response_timeout=15, resp_chunk_size=None):
"""
Get object directly from the object server.
@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
:param obj: object name
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:returns: object
:param resp_chunk_size: if defined, chunk size of data to read.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
"""
path = '/%s/%s/%s' % (account, container, obj)
with Timeout(conn_timeout):
@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
with Timeout(response_timeout):
resp = conn.getresponse()
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException(
'Object server %s:%s direct GET %s gave status %s' %
(node['ip'], node['port'],
@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
metadata = {}
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return (resp.getheader('content-type'),
int(resp.getheader('content-length')),
resp.getheader('last-modified'),
resp.getheader('etag').strip('"'),
metadata,
resp.read())
if resp_chunk_size:
def _object_body():
buf = resp.read(resp_chunk_size)
while buf:
yield buf
buf = resp.read(resp_chunk_size)
object_body = _object_body()
else:
object_body = resp.read()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def direct_delete_object(node, part, account, container, obj,
@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj,
'DELETE', path, headers)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException(
'Object server %s:%s direct DELETE %s gave status %s' %
@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj,
http_host=node['ip'], http_port=node['port'],
http_device=node['device'], http_status=resp.status,
http_reason=resp.reason)
return resp
def retry(func, *args, **kwargs):

160
swift/common/middleware/acl.py Normal file
View File

@ -0,0 +1,160 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from urlparse import urlparse
def clean_acl(name, value):
"""
Returns a cleaned ACL header value, validating that it meets the formatting
requirements for standard Swift ACL strings.
The ACL format is::
[item[,item...]]
Each item can be a group name to give access to or a referrer designation
to grant or deny based on the HTTP Referer header.
The referrer designation format is::
.r:[-]value
The ``.r`` can also be ``.ref``, ``.referer``, or ``.referrer``; though it
will be shortened to just ``.r`` for decreased character count usage.
The value can be ``*`` to specify any referrer host is allowed access, a
specific host name like ``www.example.com``, or if it has a leading period
``.`` or leading ``*.`` it is a domain name specification, like
``.example.com`` or ``*.example.com``. The leading minus sign ``-``
indicates referrer hosts that should be denied access.
Referrer access is applied in the order they are specified. For example,
.r:.example.com,.r:-thief.example.com would allow all hosts ending with
.example.com except for the specific host thief.example.com.
Example valid ACLs::
.r:*
.r:*,.r:-.thief.com
.r:*,.r:.example.com,.r:-thief.example.com
.r:*,.r:-.thief.com,bobs_account,sues_account:sue
bobs_account,sues_account:sue
Example invalid ACLs::
.r:
.r:-
Also, .r designations aren't allowed in headers whose names include the
word 'write'.
ACLs that are "messy" will be cleaned up. Examples:
====================== ======================
Original Cleaned
---------------------- ----------------------
``bob, sue`` ``bob,sue``
``bob , sue`` ``bob,sue``
``bob,,,sue`` ``bob,sue``
``.referrer : *`` ``.r:*``
``.ref:*.example.com`` ``.r:.example.com``
====================== ======================
:param name: The name of the header being cleaned, such as X-Container-Read
or X-Container-Write.
:param value: The value of the header being cleaned.
:returns: The value, cleaned of extraneous formatting.
:raises ValueError: If the value does not meet the ACL formatting
requirements; the error message will indicate why.
"""
name = name.lower()
values = []
for raw_value in value.split(','):
raw_value = raw_value.strip()
if raw_value:
if ':' not in raw_value:
values.append(raw_value)
else:
first, second = (v.strip() for v in raw_value.split(':', 1))
if not first or first[0] != '.':
values.append(raw_value)
elif first in ('.r', '.ref', '.referer', '.referrer'):
if 'write' in name:
raise ValueError('Referrers not allowed in write ACL: '
'%s' % repr(raw_value))
negate = False
if second and second[0] == '-':
negate = True
second = second[1:].strip()
if second and second != '*' and second[0] == '*':
second = second[1:].strip()
if not second or second == '.':
raise ValueError('No host/domain value after referrer '
'designation in ACL: %s' % repr(raw_value))
values.append('.r:%s%s' % (negate and '-' or '', second))
else:
raise ValueError('Unknown designator %s in ACL: %s' %
(repr(first), repr(raw_value)))
return ','.join(values)
def parse_acl(acl_string):
"""
Parses a standard Swift ACL string into a referrers list and groups list.
See :func:`clean_acl` for documentation of the standard Swift ACL format.
:param acl_string: The standard Swift ACL string to parse.
:returns: A tuple of (referrers, groups) where referrers is a list of
referrer designations (without the leading .r:) and groups is a
list of groups to allow access.
"""
referrers = []
groups = []
if acl_string:
for value in acl_string.split(','):
if value.startswith('.r:'):
referrers.append(value[len('.r:'):])
else:
groups.append(value)
return referrers, groups
def referrer_allowed(referrer, referrer_acl):
"""
Returns True if the referrer should be allowed based on the referrer_acl
list (as returned by :func:`parse_acl`).
See :func:`clean_acl` for documentation of the standard Swift ACL format.
:param referrer: The value of the HTTP Referer header.
:param referrer_acl: The list of referrer designations as returned by
:func:`parse_acl`.
:returns: True if the referrer should be allowed; False if not.
"""
allow = False
if referrer_acl:
rhost = urlparse(referrer or '').hostname or 'unknown'
for mhost in referrer_acl:
if mhost[0] == '-':
mhost = mhost[1:]
if mhost == rhost or \
(mhost[0] == '.' and rhost.endswith(mhost)):
allow = False
elif mhost == '*' or mhost == rhost or \
(mhost[0] == '.' and rhost.endswith(mhost)):
allow = True
return allow

145
swift/common/middleware/auth.py
View File

@ -13,31 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from time import time
from webob.request import Request
from webob.exc import HTTPUnauthorized, HTTPPreconditionFailed
from eventlet.timeout import Timeout
from webob.exc import HTTPForbidden, HTTPUnauthorized
from swift.common.utils import split_path
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.utils import get_logger, cache_from_env
from swift.common.memcached import MemcacheRing
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
from swift.common.utils import cache_from_env, split_path
class DevAuthMiddleware(object):
"""
Auth Middleware that uses the dev auth server
"""
class DevAuth(object):
"""Auth Middleware that uses the dev auth server."""
def __init__(self, app, conf, memcache_client=None, logger=None):
def __init__(self, app, conf):
self.app = app
self.memcache_client = memcache_client
if logger is None:
self.logger = get_logger(conf)
else:
self.logger = logger
self.conf = conf
self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
if self.reseller_prefix and self.reseller_prefix[-1] != '_':
self.reseller_prefix += '_'
self.auth_host = conf.get('ip', '127.0.0.1')
self.auth_port = int(conf.get('port', 11000))
self.ssl = \
@ -45,68 +39,79 @@ class DevAuthMiddleware(object):
self.timeout = int(conf.get('node_timeout', 10))
def __call__(self, env, start_response):
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
req = Request(env)
if 'x-storage-token' in req.headers and \
'x-auth-token' not in req.headers:
req.headers['x-auth-token'] = req.headers['x-storage-token']
try:
version, account, container, obj = split_path(req.path, 1, 4, True)
except ValueError, e:
version = account = container = obj = None
if account is None:
return HTTPPreconditionFailed(request=req, body='Bad URL')(
env, start_response)
if not req.headers.get('x-auth-token'):
return HTTPPreconditionFailed(request=req,
body='Missing Auth Token')(env, start_response)
if not self.auth(account, req.headers['x-auth-token']):
return HTTPUnauthorized(request=req)(env, start_response)
# If we get here, then things should be good.
"""
Accepts a standard WSGI application call, authenticating the request
and installing callback hooks for authorization and ACL header
validation. For an authenticated request, REMOTE_USER will be set to a
comma separated list of the user's groups.
"""
groups = None
token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
if token and token.startswith(self.reseller_prefix):
memcache_client = cache_from_env(env)
key = 'devauth/%s' % token
cached_auth_data = memcache_client.get(key)
if cached_auth_data:
start, expiration, groups = cached_auth_data
if time() - start > expiration:
groups = None
if not groups:
with Timeout(self.timeout):
conn = http_connect(self.auth_host, self.auth_port, 'GET',
'/token/%s' % token, ssl=self.ssl)
resp = conn.getresponse()
resp.read()
conn.close()
if resp.status // 100 != 2:
return HTTPUnauthorized()(env, start_response)
expiration = float(resp.getheader('x-auth-ttl'))
groups = resp.getheader('x-auth-groups')
memcache_client.set(key, (time(), expiration, groups),
timeout=expiration)
env['REMOTE_USER'] = groups
env['swift.authorize'] = self.authorize
env['swift.clean_acl'] = clean_acl
# We know the proxy logs the token, so we augment it just a bit to also
# log the authenticated user.
user = groups and groups.split(',', 1)[0] or ''
env['HTTP_X_AUTH_TOKEN'] = '%s,%s' % (user, token)
return self.app(env, start_response)
def auth(self, account, token):
def authorize(self, req):
"""
Dev authorization implmentation
:param account: account name
:param token: auth token
:returns: True if authorization is successful, False otherwise
Returns None if the request is authorized to continue or a standard
WSGI response callable if not.
"""
key = 'auth/%s/%s' % (account, token)
now = time.time()
cached_auth_data = self.memcache_client.get(key)
if cached_auth_data:
start, expiration = cached_auth_data
if now - start <= expiration:
return True
try:
with Timeout(self.timeout):
conn = http_connect(self.auth_host, self.auth_port, 'GET',
'/token/%s/%s' % (account, token), ssl=self.ssl)
resp = conn.getresponse()
resp.read()
conn.close()
if resp.status == 204:
validated = float(resp.getheader('x-auth-ttl'))
else:
validated = False
except:
self.logger.exception('ERROR with auth')
return False
if not validated:
return False
version, account, container, obj = split_path(req.path, 1, 4, True)
if not account or not account.startswith(self.reseller_prefix):
return self.denied_response(req)
if req.remote_user and account in req.remote_user.split(','):
return None
referrers, groups = parse_acl(getattr(req, 'acl', None))
if referrer_allowed(req.referer, referrers):
return None
if not req.remote_user:
return self.denied_response(req)
for user_group in req.remote_user.split(','):
if user_group in groups:
return None
return self.denied_response(req)
def denied_response(self, req):
"""
Returns a standard WSGI response callable with the status of 403 or 401
depending on whether the REMOTE_USER is set or not.
"""
if req.remote_user:
return HTTPForbidden(request=req)
else:
val = (now, validated)
self.memcache_client.set(key, val, timeout=validated)
return True
return HTTPUnauthorized(request=req)
def filter_factory(global_conf, **local_conf):
"""Returns a WSGI filter app for use with paste.deploy."""
conf = global_conf.copy()
conf.update(local_conf)
def auth_filter(app):
return DevAuthMiddleware(app, conf)
return DevAuth(app, conf)
return auth_filter

9
swift/container/server.py
View File

@ -44,6 +44,9 @@ DATADIR = 'containers'
class ContainerController(object):
"""WSGI Controller for the container server."""
# Ensure these are all lowercase
save_headers = ['x-container-read', 'x-container-write']
def __init__(self, conf):
self.logger = get_logger(conf)
self.root = conf.get('devices', '/srv/node/')
@ -192,7 +195,8 @@ class ContainerController(object):
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-container-meta-'))
if key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-'))
if metadata:
broker.update_metadata(metadata)
resp = self.account_update(req, account, container, broker)
@ -373,7 +377,8 @@ class ContainerController(object):
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-container-meta-'))
if key.lower() in self.save_headers or
key.lower().startswith('x-container-meta-'))
if metadata:
broker.update_metadata(metadata)
return HTTPNoContent(request=req)

161
swift/proxy/server.py
View File

@ -17,6 +17,7 @@ from __future__ import with_statement
import mimetypes
import os
import time
import traceback
from ConfigParser import ConfigParser
from urllib import unquote, quote
import uuid
@ -73,6 +74,22 @@ def public(func):
return wrapped
def delay_denial(func):
"""
Decorator to declare which methods should have any swift.authorize call
delayed. This is so the method can load the Request object up with
additional information that may be needed by the authorization system.
:param func: function to delay authorization on
"""
func.delay_denial = True
@functools.wraps(func)
def wrapped(*a, **kw):
return func(*a, **kw)
return wrapped
class Controller(object):
"""Base WSGI controller class for the proxy"""
@ -206,19 +223,28 @@ class Controller(object):
:param account: account name for the container
:param container: container name to look up
:returns: tuple of (container partition, container nodes) or
(None, None) if the container does not exist
:returns: tuple of (container partition, container nodes, container
read acl, container write acl) or (None, None, None, None) if
the container does not exist
"""
partition, nodes = self.app.container_ring.get_nodes(
account, container)
path = '/%s/%s' % (account, container)
cache_key = 'container%s' % path
# Older memcache values (should be treated as if they aren't there):
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache.get(cache_key) == 200:
return partition, nodes
# Newer memcache values:
# [older status value from above, read acl, write acl]
cache_value = self.app.memcache.get(cache_key)
if hasattr(cache_value, '__iter__'):
status, read_acl, write_acl = cache_value
if status == 200:
return partition, nodes, read_acl, write_acl
if not self.account_info(account)[1]:
return (None, None)
return (None, None, None, None)
result_code = 0
read_acl = None
write_acl = None
attempts_left = self.app.container_ring.replica_count
headers = {'x-cf-trans-id': self.trans_id}
for node in self.iter_nodes(partition, nodes, self.app.container_ring):
@ -233,6 +259,8 @@ class Controller(object):
body = resp.read()
if 200 <= resp.status <= 299:
result_code = 200
read_acl = resp.getheader('x-container-read')
write_acl = resp.getheader('x-container-write')
break
elif resp.status == 404:
result_code = 404 if not result_code else -1
@ -251,10 +279,11 @@ class Controller(object):
cache_timeout = self.app.recheck_container_existence
else:
cache_timeout = self.app.recheck_container_existence * 0.1
self.app.memcache.set(cache_key, result_code, timeout=cache_timeout)
self.app.memcache.set(cache_key, (result_code, read_acl, write_acl),
timeout=cache_timeout)
if result_code == 200:
return partition, nodes
return (None, None)
return partition, nodes, read_acl, write_acl
return (None, None, None, None)
def iter_nodes(self, partition, nodes, ring):
"""
@ -474,6 +503,12 @@ class ObjectController(Controller):
def GETorHEAD(self, req):
"""Handle HTTP GET or HEAD requests."""
if 'swift.authorize' in req.environ:
req.acl = \
self.container_info(self.account_name, self.container_name)[2]
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
partition, nodes = self.app.object_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
return self.GETorHEAD_base(req, 'Object', partition,
@ -481,13 +516,30 @@ class ObjectController(Controller):
req.path_info, self.app.object_ring.replica_count)
@public
@delay_denial
def GET(self, req):
"""Handler for HTTP GET requests."""
return self.GETorHEAD(req)
@public
@delay_denial
def HEAD(self, req):
"""Handler for HTTP HEAD requests."""
return self.GETorHEAD(req)
@public
@delay_denial
def POST(self, req):
"""HTTP POST request handler."""
error_response = check_metadata(req, 'object')
if error_response:
return error_response
container_partition, containers = \
container_partition, containers, _, req.acl = \
self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
containers = self.get_update_nodes(container_partition, containers,
@ -521,10 +573,15 @@ class ObjectController(Controller):
bodies, 'Object POST')
@public
@delay_denial
def PUT(self, req):
"""HTTP PUT request handler."""
container_partition, containers = \
container_partition, containers, _, req.acl = \
self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
containers = self.get_update_nodes(container_partition, containers,
@ -701,10 +758,15 @@ class ObjectController(Controller):
return resp
@public
@delay_denial
def DELETE(self, req):
"""HTTP DELETE request handler."""
container_partition, containers = \
container_partition, containers, _, req.acl = \
self.container_info(self.account_name, self.container_name)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
containers = self.get_update_nodes(container_partition, containers,
@ -771,11 +833,26 @@ class ObjectController(Controller):
class ContainerController(Controller):
"""WSGI controller for container requests"""
# Ensure these are all lowercase
pass_through_headers = ['x-container-read', 'x-container-write']
def __init__(self, app, account_name, container_name, **kwargs):
Controller.__init__(self, app)
self.account_name = unquote(account_name)
self.container_name = unquote(container_name)
def clean_acls(self, req):
if 'swift.clean_acl' in req.environ:
for header in ('x-container-read', 'x-container-write'):
if header in req.headers:
try:
req.headers[header] = \
req.environ['swift.clean_acl'](header,
req.headers[header])
except ValueError, err:
return HTTPBadRequest(request=req, body=str(err))
return None
def GETorHEAD(self, req):
"""Handler for HTTP GET/HEAD requests."""
if not self.account_info(self.account_name)[1]:
@ -784,12 +861,30 @@ class ContainerController(Controller):
self.account_name, self.container_name)
resp = self.GETorHEAD_base(req, 'Container', part, nodes,
req.path_info, self.app.container_ring.replica_count)
if 'swift.authorize' in req.environ:
req.acl = resp.headers.get('x-container-read')
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
return resp
@public
@delay_denial
def GET(self, req):
"""Handler for HTTP GET requests."""
return self.GETorHEAD(req)
@public
@delay_denial
def HEAD(self, req):
"""Handler for HTTP HEAD requests."""
return self.GETorHEAD(req)
@public
def PUT(self, req):
"""HTTP PUT request handler."""
error_response = check_metadata(req, 'container')
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
@ -807,7 +902,8 @@ class ContainerController(Controller):
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'x-cf-trans-id': self.trans_id}
headers.update(value for value in req.headers.iteritems()
if value[0].lower().startswith('x-container-meta-'))
if value[0].lower() in self.pass_through_headers or
value[0].lower().startswith('x-container-meta-'))
statuses = []
reasons = []
bodies = []
@ -853,7 +949,8 @@ class ContainerController(Controller):
@public
def POST(self, req):
"""HTTP POST request handler."""
error_response = check_metadata(req, 'container')
error_response = \
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
account_partition, accounts = self.account_info(self.account_name)
@ -864,7 +961,8 @@ class ContainerController(Controller):
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'x-cf-trans-id': self.trans_id}
headers.update(value for value in req.headers.iteritems()
if value[0].lower().startswith('x-container-meta-'))
if value[0].lower() in self.pass_through_headers or
value[0].lower().startswith('x-container-meta-'))
statuses = []
reasons = []
bodies = []
@ -1118,7 +1216,8 @@ class BaseApplication(object):
self.posthooklogger(env, req)
return response
except:
print "EXCEPTION IN __call__: %s" % env
print "EXCEPTION IN __call__: %s: %s" % \
(traceback.format_exc(), env)
start_response('500 Server Error',
[('Content-Type', 'text/plain')])
return ['Internal server error.\n']
@ -1160,12 +1259,30 @@ class BaseApplication(object):
controller.trans_id = req.headers.get('x-cf-trans-id', '-')
try:
handler = getattr(controller, req.method)
if getattr(handler, 'publicly_accessible'):
if path_parts['version']:
req.path_info_pop()
return handler(req)
if not getattr(handler, 'publicly_accessible'):
handler = None
except AttributeError:
handler = None
if not handler:
return HTTPMethodNotAllowed(request=req)
if path_parts['version']:
req.path_info_pop()
if 'swift.authorize' in req.environ:
# We call authorize before the handler, always. If authorized,
# we remove the swift.authorize hook so isn't ever called
# again. If not authorized, we return the denial unless the
# controller's method indicates it'd like to gather more
# information and try again later.
resp = req.environ['swift.authorize'](req)
if not resp:
# No resp means authorized, no delayed recheck required.
del req.environ['swift.authorize']
else:
# Response indicates denial, but we might delay the denial
# and recheck later. If not delayed, return the error now.
if not getattr(handler, 'delay_denial', None):
return resp
return handler(req)
except Exception:
self.logger.exception('ERROR Unhandled exception in request')
return HTTPServerError(request=req)
@ -1187,7 +1304,9 @@ class Application(BaseApplication):
return req.response
def posthooklogger(self, env, req):
response = req.response
response = getattr(req, 'response', None)
if not response:
return
trans_time = '%.4f' % (time.time() - req.start_time)
the_request = quote(unquote(req.path))
if req.query_string:

20
test/functional/sample.conf
View File

@ -1,10 +1,20 @@
# Sample functional test configuration file
# sample config
auth_host = 127.0.0.1
auth_port = 80
auth_port = 11000
auth_ssl = no
account = test_account
username = test_user
password = test_password
# Primary functional test account (needs admin access to the account)
account = test
username = tester
password = testing
# User on a second account (needs admin access to the account)
account2 = test2
username2 = tester2
password2 = testing2
# User on same account as first, but without admin access
username3 = tester3
password3 = testing3
collate = C

16
test/functional/tests.py
View File

@ -106,9 +106,12 @@ class Base(unittest.TestCase):
self.assert_(response_body == body,
'Body returned: %s' % (response_body))
def assert_status(self, status):
self.assert_(self.env.conn.response.status == status,
'Status returned: %d' % (self.env.conn.response.status))
def assert_status(self, status_or_statuses):
self.assert_(self.env.conn.response.status == status_or_statuses or
(hasattr(status_or_statuses, '__iter__') and
self.env.conn.response.status in status_or_statuses),
'Status returned: %d Expected: %s' %
(self.env.conn.response.status, status_or_statuses))
class Base2(object):
def setUp(self):
@ -148,11 +151,11 @@ class TestAccount(Base):
def testNoAuthToken(self):
self.assertRaises(ResponseError, self.env.account.info,
cfg={'no_auth_token':True})
self.assert_status(412)
self.assert_status([401, 412])
self.assertRaises(ResponseError, self.env.account.containers,
cfg={'no_auth_token':True})
self.assert_status(412)
self.assert_status([401, 412])
def testInvalidUTF8Path(self):
invalid_utf8 = Utils.create_utf8_name()[::-1]
@ -1123,7 +1126,8 @@ class TestFile(Base):
self.assert_status(400)
# bad request types
for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'):
#for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'):
for req in ('LICK', 'GETorHEAD_base'):
self.env.account.conn.make_request(req)
self.assert_status(405)

67
test/functionalnosetests/swift_testing.py
View File

@ -10,11 +10,11 @@ from swift.common.client import get_auth, http_connection
swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
swift_test_user = os.environ.get('SWIFT_TEST_USER')
swift_test_key = os.environ.get('SWIFT_TEST_KEY')
swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None]
swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None]
# If no environment set, fall back to old school conf file
if not all([swift_test_auth, swift_test_user, swift_test_key]):
if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]):
conf = ConfigParser()
class Sectionizer(object):
def __init__(self, fp):
@ -32,16 +32,36 @@ if not all([swift_test_auth, swift_test_user, swift_test_key]):
if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'):
swift_test_auth = 'https'
swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf
swift_test_user = '%(account)s:%(username)s' % conf
swift_test_key = conf['password']
swift_test_user[0] = '%(account)s:%(username)s' % conf
swift_test_key[0] = conf['password']
try:
swift_test_user[1] = '%(account2)s:%(username2)s' % conf
swift_test_key[1] = conf['password2']
except KeyError, err:
pass # old conf, no second account tests can be run
try:
swift_test_user[2] = '%(account)s:%(username3)s' % conf
swift_test_key[2] = conf['password3']
except KeyError, err:
pass # old conf, no third account tests can be run
except IOError, err:
if err.errno != errno.ENOENT:
raise
skip = not all([swift_test_auth, swift_test_user, swift_test_key])
skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
if skip:
print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'
skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
if not skip and skip2:
print >>sys.stderr, \
'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
if not skip and skip3:
print >>sys.stderr, \
'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
class AuthError(Exception):
pass
@ -51,29 +71,44 @@ class InternalServerError(Exception):
pass
url = token = parsed = conn = None
url = [None, None, None]
token = [None, None, None]
parsed = [None, None, None]
conn = [None, None, None]
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override the 'retries' (default: 5) and
'use_account' (default: 1).
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
use_account = 1
if 'use_account' in kwargs:
use_account = kwargs['use_account']
del kwargs['use_account']
use_account -= 1
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
if not url or not token:
url, token = \
get_auth(swift_test_auth, swift_test_user, swift_test_key)
parsed = conn = None
if not parsed or not conn:
parsed, conn = http_connection(url)
return func(url, token, parsed, conn, *args, **kwargs)
if not url[use_account] or not token[use_account]:
url[use_account], token[use_account] = \
get_auth(swift_test_auth, swift_test_user[use_account],
swift_test_key[use_account])
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account], *args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise
parsed = conn = None
parsed[use_account] = conn[use_account] = None
except AuthError, err:
url = token = None
url[use_account] = token[use_account] = None
continue
except InternalServerError, err:
pass

7
test/functionalnosetests/test_account.py
View File

@ -1,6 +1,7 @@
#!/usr/bin/python
import unittest
from nose import SkipTest
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
@ -12,7 +13,7 @@ class TestAccount(unittest.TestCase):
def test_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, value):
conn.request('POST', parsed.path, '',
{'X-Auth-Token': token, 'X-Account-Meta-Test': value})
@ -48,7 +49,7 @@ class TestAccount(unittest.TestCase):
def test_multi_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, name, value):
conn.request('POST', parsed.path, '',
{'X-Auth-Token': token, name: value})
@ -74,7 +75,7 @@ class TestAccount(unittest.TestCase):
def test_bad_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)

237
test/functionalnosetests/test_container.py
View File

@ -1,19 +1,22 @@
#!/usr/bin/python
import json
import unittest
from nose import SkipTest
from uuid import uuid4
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
from swift_testing import check_response, retry, skip
from swift_testing import check_response, retry, skip, skip2, skip3, \
swift_test_user
class TestContainer(unittest.TestCase):
def setUp(self):
if skip:
return
raise SkipTest
self.name = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.name, '',
@ -25,7 +28,27 @@ class TestContainer(unittest.TestCase):
def tearDown(self):
if skip:
return
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name + '?format=json',
'', {'X-Auth-Token': token})
return check_response(conn)
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'/'.join([parsed.path, self.name, obj['name']]), '',
{'X-Auth-Token': token})
return check_response(conn)
while True:
resp = retry(get)
body = resp.read()
self.assert_(resp.status // 100 == 2, resp.status)
objs = json.loads(body)
if not objs:
break
for obj in objs:
resp = retry(delete, obj)
resp.read()
self.assertEquals(resp.status, 204)
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
@ -36,7 +59,7 @@ class TestContainer(unittest.TestCase):
def test_multi_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, name, value):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, name: value})
@ -63,7 +86,7 @@ class TestContainer(unittest.TestCase):
def test_PUT_metadata(self):
if skip:
return
raise SkipTest
def put(url, token, parsed, conn, name, value):
conn.request('PUT', parsed.path + '/' + name, '',
{'X-Auth-Token': token, 'X-Container-Meta-Test': value})
@ -110,7 +133,7 @@ class TestContainer(unittest.TestCase):
def test_POST_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, value):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Meta-Test': value})
@ -145,7 +168,7 @@ class TestContainer(unittest.TestCase):
def test_PUT_bad_metadata(self):
if skip:
return
raise SkipTest
def put(url, token, parsed, conn, name, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
@ -240,7 +263,7 @@ class TestContainer(unittest.TestCase):
def test_POST_bad_metadata(self):
if skip:
return
raise SkipTest
def post(url, token, parsed, conn, extra_headers):
headers = {'X-Auth-Token': token}
headers.update(extra_headers)
@ -297,6 +320,204 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assertEquals(resp.status, 400)
def test_public_container(self):
if skip:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET', parsed.path + '/' + self.name)
return check_response(conn)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
self.assert_(str(err).startswith('No result after '), err)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Read': '.r:*'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(get)
resp.read()
self.assertEquals(resp.status, 204)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': ''})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
self.assert_(str(err).startswith('No result after '), err)
def test_cross_account_container(self):
if skip or skip2:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the second account
def get2(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container accessible by the second account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': 'test2',
'X-Container-Write': 'test2'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can now use the container with the second account
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 204)
# Make the container private again
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': '',
'X-Container-Write': ''})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can't access the container with the second account again
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
def test_cross_account_public_container(self):
if skip or skip2:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the second account
def get2(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container completely public
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Read': '.r:*'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can now read the container with the second account
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 204)
# But we shouldn't be able to write with the second account
def put2(url, token, parsed, conn):
conn.request('PUT', first_account[0] + '/' + self.name + '/object',
'test object', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put2, use_account=2)
resp.read()
self.assertEquals(resp.status, 403)
# Now make the container also writeable by the second account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Write': 'test2'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can still read the container with the second account
resp = retry(get2, use_account=2)
resp.read()
self.assertEquals(resp.status, 204)
# And that we can now write with the second account
resp = retry(put2, use_account=2)
resp.read()
self.assertEquals(resp.status, 201)
def test_nonadmin_user(self):
if skip or skip3:
raise SkipTest
# Obtain the first account's string
first_account = ['unknown']
def get1(url, token, parsed, conn):
first_account[0] = parsed.path
conn.request('HEAD', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get1)
resp.read()
# Ensure we can't access the container with the third account
def get3(url, token, parsed, conn):
conn.request('GET', first_account[0] + '/' + self.name, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(get3, use_account=3)
resp.read()
self.assertEquals(resp.status, 403)
# Make the container accessible by the third account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can now read the container with the third account
resp = retry(get3, use_account=3)
resp.read()
self.assertEquals(resp.status, 204)
# But we shouldn't be able to write with the third account
def put3(url, token, parsed, conn):
conn.request('PUT', first_account[0] + '/' + self.name + '/object',
'test object', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put3, use_account=3)
resp.read()
self.assertEquals(resp.status, 403)
# Now make the container also writeable by the third account
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.name, '',
{'X-Auth-Token': token,
'X-Container-Write': swift_test_user[2]})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
# Ensure we can still read the container with the third account
resp = retry(get3, use_account=3)
resp.read()
self.assertEquals(resp.status, 204)
# And that we can now write with the third account
resp = retry(put3, use_account=3)
resp.read()
self.assertEquals(resp.status, 201)
if __name__ == '__main__':
unittest.main()

91
test/functionalnosetests/test_object.py Normal file
View File

@ -0,0 +1,91 @@
#!/usr/bin/python
import unittest
from nose import SkipTest
from uuid import uuid4
from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
from swift_testing import check_response, retry, skip
class TestObject(unittest.TestCase):
def setUp(self):
if skip:
raise SkipTest
self.container = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEquals(resp.status, 201)
self.obj = uuid4().hex
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
self.obj), 'test', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(put)
resp.read()
self.assertEquals(resp.status, 201)
def tearDown(self):
if skip:
raise SkipTest
def delete(url, token, parsed, conn):
conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container,
self.obj), '', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
def test_public_object(self):
if skip:
raise SkipTest
def get(url, token, parsed, conn):
conn.request('GET',
'%s/%s/%s' % (parsed.path, self.container, self.obj))
return check_response(conn)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
self.assert_(str(err).startswith('No result after '))
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token,
'X-Container-Read': '.r:*'})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
resp = retry(get)
resp.read()
self.assertEquals(resp.status, 200)
def post(url, token, parsed, conn):
conn.request('POST', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token, 'X-Container-Read': ''})
return check_response(conn)
resp = retry(post)
resp.read()
self.assertEquals(resp.status, 204)
try:
resp = retry(get)
raise Exception('Should not have been able to GET')
except Exception, err:
self.assert_(str(err).startswith('No result after '))
if __name__ == '__main__':
unittest.main()

46
test/probe/test_account_failures.py
View File

@ -39,8 +39,10 @@ class TestAccountFailures(unittest.TestCase):
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for c in containers:
@ -56,8 +58,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for c in containers:
@ -73,8 +77,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
get_to_final_state()
containers = client.get_account(self.url, self.token)
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -94,8 +100,10 @@ class TestAccountFailures(unittest.TestCase):
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -115,8 +123,10 @@ class TestAccountFailures(unittest.TestCase):
'once']))
for p in ps:
p.wait()
self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for c in containers:
@ -134,10 +144,12 @@ class TestAccountFailures(unittest.TestCase):
'/etc/swift/account-server/%d.conf' %
((anodes[0]['port'] - 6002) / 10)]).pid
sleep(2)
# This is the earlier object count and bytes because the first node
# doesn't have the newest udpates yet.
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
containers = client.get_account(self.url, self.token)
# This is the earlier counts and bytes because the first node doesn't
# have the newest udpates yet.
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -155,8 +167,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
get_to_final_state()
containers = client.get_account(self.url, self.token)
self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for c in containers:

104
test/probe/test_container_failures.py
View File

@ -40,23 +40,23 @@ class TestContainerFailures(unittest.TestCase):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
@ -64,11 +64,11 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -87,7 +87,7 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
@ -95,44 +95,44 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_second_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[1]['port']]] = \
Popen(['swift-container-server',
@ -140,9 +140,9 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[1]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -161,42 +161,42 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
@ -204,9 +204,9 @@ class TestContainerFailures(unittest.TestCase):
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
@ -215,11 +215,11 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -238,7 +238,7 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
@ -246,35 +246,35 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_last_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in (1, 2):
@ -282,9 +282,9 @@ class TestContainerFailures(unittest.TestCase):
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
for x in (1, 2):
self.pids[self.port2server[cnodes[x]['port']]] = \
@ -293,9 +293,9 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -314,29 +314,29 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
if __name__ == '__main__':

4
test/probe/test_object_async_update.py
View File

@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
((cnode['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(not direct_client.direct_get_container(cnode, cpart,
self.account, container))
self.account, container)[1])
ps = []
for n in xrange(1, 5):
ps.append(Popen(['swift-object-updater',
@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
for p in ps:
p.wait()
objs = [o['name'] for o in direct_client.direct_get_container(cnode,
cpart, self.account, container)]
cpart, self.account, container)[1]]
self.assert_(obj in objs)

29
test/probe/test_object_handoff.py
View File

@ -75,13 +75,13 @@ class TestObjectHandoff(unittest.TestCase):
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)]
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(cnode, cpart,
self.account, container)]
self.account, container)[1]]
if obj not in objs:
raise Exception(
'Container server %s:%s did not know about object' %
@ -126,11 +126,10 @@ class TestObjectHandoff(unittest.TestCase):
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.post_object(self.url, self.token, container, obj,
{'probe': 'value'})
ometadata = client.head_object(
self.url, self.token, container, obj)[-1]
if ometadata.get('probe') != 'value':
raise Exception('Metadata incorrect, was %s' % repr(ometadata))
headers={'x-object-meta-probe': 'value'})
oheaders = client.head_object(self.url, self.token, container, obj)
if oheaders.get('x-object-meta-probe') != 'value':
raise Exception('Metadata incorrect, was %s' % repr(oheaders))
exc = False
try:
direct_client.direct_get_object(another_onode, opart, self.account,
@ -145,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase):
'/etc/swift/object-server/%d.conf' %
((onode['port'] - 6000) / 10)]).pid
sleep(2)
ometadata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-2]
if ometadata.get('probe') == 'value':
oheaders = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[0]
if oheaders.get('x-object-meta-probe') == 'value':
raise Exception('Previously downed object server had the new '
'metadata when it should not have it')
# Run the extra server last so it'll remove it's extra partition
@ -161,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase):
call(['swift-object-replicator',
'/etc/swift/object-server/%d.conf' %
((another_onode['port'] - 6000) / 10), 'once'])
ometadata = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[-2]
if ometadata.get('probe') != 'value':
oheaders = direct_client.direct_get_object(onode, opart, self.account,
container, obj)[0]
if oheaders.get('x-object-meta-probe') != 'value':
raise Exception(
'Previously downed object server did not have the new metadata')
@ -177,13 +176,13 @@ class TestObjectHandoff(unittest.TestCase):
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)]
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:
objs = [o['name'] for o in
direct_client.direct_get_container(
cnode, cpart, self.account, container)]
cnode, cpart, self.account, container)[1]]
if obj in objs:
raise Exception(
'Container server %s:%s still knew about object' %

27
test/probe/test_running_with_each_type_down.py
View File

@ -56,16 +56,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
pass
client.put_object(self.url, self.token, 'container1', 'object1', '1234')
get_to_final_state()
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
@ -84,15 +87,18 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
'/etc/swift/object-server/%d.conf' %
((onodes[0]['port'] - 6000) / 10)]).pid
sleep(2)
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
# The account node was previously down.
self.assert_(not found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
@ -101,16 +107,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
self.assert_(found1)
get_to_final_state()
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)

179
test/unit/auth/test_server.py
View File

@ -21,10 +21,11 @@ from StringIO import StringIO
from uuid import uuid4
from logging import StreamHandler
import sqlite3
from webob import Request
from swift.auth import server as auth_server
from swift.common.db import DatabaseConnectionError
from swift.common.db import DatabaseConnectionError, get_db_connection
from swift.common.utils import get_logger
@ -106,38 +107,25 @@ class TestAuthServer(unittest.TestCase):
def test_validate_token_non_existant_token(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing',).split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
self.assertEquals(self.controller.validate_token(token + 'bad',
cfaccount), False)
def test_validate_token_non_existant_cfaccount(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
self.assertEquals(self.controller.validate_token(token,
cfaccount + 'bad'), False)
self.assertEquals(self.controller.validate_token(token + 'bad'), False)
def test_validate_token_good(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing',).split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
def test_validate_token_expired(self):
@ -145,40 +133,38 @@ class TestAuthServer(unittest.TestCase):
try:
auth_server.time = lambda: 1
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account('test', 'tester',
cfaccount = self.controller.create_user('test', 'tester',
'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(
token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
auth_server.time = lambda: 1 + self.controller.token_life
self.assertEquals(self.controller.validate_token(
token, cfaccount), False)
self.assertEquals(self.controller.validate_token(token), False)
finally:
auth_server.time = orig_time
def test_create_account_no_new_account(self):
def test_create_user_no_new_account(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
result = self.controller.create_account('', 'tester', 'testing')
result = self.controller.create_user('', 'tester', 'testing')
self.assertFalse(result)
def test_create_account_no_new_user(self):
def test_create_user_no_new_user(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
result = self.controller.create_account('test', '', 'testing')
result = self.controller.create_user('test', '', 'testing')
self.assertFalse(result)
def test_create_account_no_new_password(self):
def test_create_user_no_new_password(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
result = self.controller.create_account('test', 'tester', '')
result = self.controller.create_user('test', 'tester', '')
self.assertFalse(result)
def test_create_account_good(self):
def test_create_user_good(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test', 'tester', 'testing')
url = self.controller.create_user('test', 'tester', 'testing')
self.assert_(url)
self.assertEquals('/'.join(url.split('/')[:-1]),
self.controller.default_cluster_url.rstrip('/'), repr(url))
@ -191,7 +177,7 @@ class TestAuthServer(unittest.TestCase):
def test_recreate_accounts_one(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_account('test', 'tester', 'testing')
self.controller.create_user('test', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
rv = self.controller.recreate_accounts()
self.assertEquals(rv.split()[0], '1', repr(rv))
@ -199,13 +185,13 @@ class TestAuthServer(unittest.TestCase):
def test_recreate_accounts_several(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_account('test1', 'tester', 'testing')
self.controller.create_user('test1', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_account('test2', 'tester', 'testing')
self.controller.create_user('test2', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_account('test3', 'tester', 'testing')
self.controller.create_user('test3', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_account('test4', 'tester', 'testing')
self.controller.create_user('test4', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201,
201, 201, 201,
201, 201, 201,
@ -216,7 +202,7 @@ class TestAuthServer(unittest.TestCase):
def test_recreate_accounts_one_fail(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test', 'tester', 'testing')
url = self.controller.create_user('test', 'tester', 'testing')
cfaccount = url.split('/')[-1]
auth_server.http_connect = fake_http_connect(500, 500, 500)
rv = self.controller.recreate_accounts()
@ -226,16 +212,16 @@ class TestAuthServer(unittest.TestCase):
def test_recreate_accounts_several_fail(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test1', 'tester', 'testing')
url = self.controller.create_user('test1', 'tester', 'testing')
cfaccounts = [url.split('/')[-1]]
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test2', 'tester', 'testing')
url = self.controller.create_user('test2', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test3', 'tester', 'testing')
url = self.controller.create_user('test3', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test4', 'tester', 'testing')
url = self.controller.create_user('test4', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(500, 500, 500,
500, 500, 500,
@ -244,20 +230,20 @@ class TestAuthServer(unittest.TestCase):
rv = self.controller.recreate_accounts()
self.assertEquals(rv.split()[0], '4', repr(rv))
failed = rv.split('[', 1)[-1][:-1].split(', ')
self.assertEquals(failed, [repr(a) for a in cfaccounts])
self.assertEquals(set(failed), set(repr(a) for a in cfaccounts))
def test_recreate_accounts_several_fail_some(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test1', 'tester', 'testing')
url = self.controller.create_user('test1', 'tester', 'testing')
cfaccounts = [url.split('/')[-1]]
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test2', 'tester', 'testing')
url = self.controller.create_user('test2', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test3', 'tester', 'testing')
url = self.controller.create_user('test3', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test4', 'tester', 'testing')
url = self.controller.create_user('test4', 'tester', 'testing')
cfaccounts.append(url.split('/')[-1])
auth_server.http_connect = fake_http_connect(500, 500, 500,
201, 201, 201,
@ -266,11 +252,8 @@ class TestAuthServer(unittest.TestCase):
rv = self.controller.recreate_accounts()
self.assertEquals(rv.split()[0], '4', repr(rv))
failed = rv.split('[', 1)[-1][:-1].split(', ')
expected = []
for i, value in enumerate(cfaccounts):
if not i % 2:
expected.append(repr(value))
self.assertEquals(failed, expected)
self.assertEquals(
len(set(repr(a) for a in cfaccounts) - set(failed)), 2)
def test_auth_bad_path(self):
self.assertRaises(ValueError, self.controller.handle_auth,
@ -281,7 +264,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_SOSO_missing_headers(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -297,7 +280,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_SOSO_bad_account(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/testbad/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -312,7 +295,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_SOSO_bad_user(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -327,7 +310,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_SOSO_bad_password(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -342,31 +325,31 @@ class TestAuthServer(unittest.TestCase):
def test_auth_SOSO_good(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
def test_auth_SOSO_good_Mosso_headers(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Auth-User': 'test:tester',
'X-Auth-Key': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
def test_auth_SOSO_bad_Mosso_headers(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing',).split('/')[-1]
res = self.controller.handle_auth(Request.blank('/v1/test/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -386,7 +369,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_missing_headers(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'}))
@ -402,7 +385,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_bad_header_format(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -417,7 +400,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_bad_account(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -432,7 +415,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_bad_user(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -447,7 +430,7 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_bad_password(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
@ -462,26 +445,26 @@ class TestAuthServer(unittest.TestCase):
def test_auth_Mosso_good(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Auth-User': 'test:tester',
'X-Auth-Key': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
def test_auth_Mosso_good_SOSO_header_names(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
cfaccount = self.controller.create_account(
cfaccount = self.controller.create_user(
'test', 'tester', 'testing').split('/')[-1]
res = self.controller.handle_auth(Request.blank('/auth',
environ={'REQUEST_METHOD': 'GET'},
headers={'X-Storage-User': 'test:tester',
'X-Storage-Pass': 'testing'}))
token = res.headers['x-storage-token']
ttl = self.controller.validate_token(token, cfaccount)
ttl = self.controller.validate_token(token)
self.assert_(ttl > 0, repr(ttl))
def test_basic_logging(self):
@ -491,10 +474,10 @@ class TestAuthServer(unittest.TestCase):
logger.logger.addHandler(log_handler)
try:
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_account('test', 'tester', 'testing')
url = self.controller.create_user('test', 'tester', 'testing')
self.assertEquals(log.getvalue().rsplit(' ', 1)[0],
"auth SUCCESS create_account('test', 'tester', _) = %s" %
repr(url))
"auth SUCCESS create_user('test', 'tester', _, False) = %s"
% repr(url))
log.truncate(0)
def start_response(*args):
pass
@ -594,6 +577,58 @@ class TestAuthServer(unittest.TestCase):
auth_server.Request = orig_Request
logger.logger.handlers.remove(log_handler)
def test_upgrading_from_db1(self):
swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex
os.mkdir(swift_dir)
try:
# Create db1
db_file = os.path.join(swift_dir, 'auth.db')
conn = get_db_connection(db_file, okay_to_create=True)
conn.execute('''CREATE TABLE IF NOT EXISTS account (
account TEXT, url TEXT, cfaccount TEXT,
user TEXT, password TEXT)''')
conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account
ON account (account)''')
conn.execute('''CREATE TABLE IF NOT EXISTS token (
cfaccount TEXT, token TEXT, created FLOAT)''')
conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount
ON token (cfaccount)''')
conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created
ON token (created)''')
conn.execute('''INSERT INTO account
(account, url, cfaccount, user, password)
VALUES ('act', 'url', 'cfa', 'usr', 'pas')''')
conn.execute('''INSERT INTO token (cfaccount, token, created)
VALUES ('cfa', 'tok', '1')''')
conn.commit()
conn.close()
# Upgrade to current db
conf = {'swift_dir': swift_dir}
controller = auth_server.AuthController(conf, FakeRing())
# Check new items exist and are correct
conn = get_db_connection(db_file)
row = conn.execute('SELECT admin FROM account').fetchone()
self.assertEquals(row[0], 't')
row = conn.execute('SELECT user FROM token').fetchone()
self.assert_(not row)
finally:
rmtree(swift_dir)
def test_create_user_twice(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.controller.create_user('test', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
self.assertEquals(
self.controller.create_user('test', 'tester', 'testing'),
'already exists')
def test_create_2users_1account(self):
auth_server.http_connect = fake_http_connect(201, 201, 201)
url = self.controller.create_user('test', 'tester', 'testing')
auth_server.http_connect = fake_http_connect(201, 201, 201)
url2 = self.controller.create_user('test', 'tester2', 'testing2')
self.assertEquals(url, url2)
if __name__ == '__main__':
unittest.main()

126
test/unit/common/middleware/test_acl.py Normal file
View File

@ -0,0 +1,126 @@
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
from swift.common.middleware import acl
class TestACL(unittest.TestCase):
def test_clean_acl(self):
value = acl.clean_acl('header', '.r:*')
self.assertEquals(value, '.r:*')
value = acl.clean_acl('header', '.r:specific.host')
self.assertEquals(value, '.r:specific.host')
value = acl.clean_acl('header', '.r:.ending.with')
self.assertEquals(value, '.r:.ending.with')
value = acl.clean_acl('header', '.r:*.ending.with')
self.assertEquals(value, '.r:.ending.with')
value = acl.clean_acl('header', '.r:-*.ending.with')
self.assertEquals(value, '.r:-.ending.with')
value = acl.clean_acl('header', '.r:one,.r:two')
self.assertEquals(value, '.r:one,.r:two')
value = acl.clean_acl('header', '.r:*,.r:-specific.host')
self.assertEquals(value, '.r:*,.r:-specific.host')
value = acl.clean_acl('header', '.r:*,.r:-.ending.with')
self.assertEquals(value, '.r:*,.r:-.ending.with')
value = acl.clean_acl('header', '.r:one,.r:-two')
self.assertEquals(value, '.r:one,.r:-two')
value = acl.clean_acl('header', '.r:one,.r:-two,account,account:user')
self.assertEquals(value, '.r:one,.r:-two,account,account:user')
value = acl.clean_acl('header', 'TEST_account')
self.assertEquals(value, 'TEST_account')
value = acl.clean_acl('header', '.ref:*')
self.assertEquals(value, '.r:*')
value = acl.clean_acl('header', '.referer:*')
self.assertEquals(value, '.r:*')
value = acl.clean_acl('header', '.referrer:*')
self.assertEquals(value, '.r:*')
value = acl.clean_acl('header',
' .r : one , ,, .r:two , .r : - three ')
self.assertEquals(value, '.r:one,.r:two,.r:-three')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.unknown:test')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:*.')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : * . ')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-*.')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : - * . ')
self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : ')
self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : ')
self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-')
self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : - ')
self.assertRaises(ValueError, acl.clean_acl, 'header',
'user , .r : - ')
self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.r:r')
def test_parse_acl(self):
self.assertEquals(acl.parse_acl(None), ([], []))
self.assertEquals(acl.parse_acl(''), ([], []))
self.assertEquals(acl.parse_acl('.r:ref1'), (['ref1'], []))
self.assertEquals(acl.parse_acl('.r:-ref1'), (['-ref1'], []))
self.assertEquals(acl.parse_acl('account:user'),
([], ['account:user']))
self.assertEquals(acl.parse_acl('account'), ([], ['account']))
self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.r:ref3,.r:-ref4'),
(['ref3', '-ref4'], ['acc1', 'acc2:usr2']))
self.assertEquals(acl.parse_acl(
'acc1,acc2:usr2,.r:ref3,acc3,acc4:usr4,.r:ref5,.r:-ref6'),
(['ref3', 'ref5', '-ref6'],
['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4']))
def test_referrer_allowed(self):
self.assert_(not acl.referrer_allowed('host', None))
self.assert_(not acl.referrer_allowed('host', []))
self.assert_(acl.referrer_allowed(None, ['*']))
self.assert_(acl.referrer_allowed('', ['*']))
self.assert_(not acl.referrer_allowed(None, ['specific.host']))
self.assert_(not acl.referrer_allowed('', ['specific.host']))
self.assert_(acl.referrer_allowed('http://www.example.com/index.html',
['.example.com']))
self.assert_(acl.referrer_allowed(
'http://user@www.example.com/index.html', ['.example.com']))
self.assert_(acl.referrer_allowed(
'http://user:pass@www.example.com/index.html', ['.example.com']))
self.assert_(acl.referrer_allowed(
'http://www.example.com:8080/index.html', ['.example.com']))
self.assert_(acl.referrer_allowed(
'http://user@www.example.com:8080/index.html', ['.example.com']))
self.assert_(acl.referrer_allowed(
'http://user:pass@www.example.com:8080/index.html',
['.example.com']))
self.assert_(acl.referrer_allowed(
'http://user:pass@www.example.com:8080', ['.example.com']))
self.assert_(acl.referrer_allowed('http://www.example.com',
['.example.com']))
self.assert_(not acl.referrer_allowed('http://thief.example.com',
['.example.com', '-thief.example.com']))
self.assert_(not acl.referrer_allowed('http://thief.example.com',
['*', '-thief.example.com']))
self.assert_(acl.referrer_allowed('http://www.example.com',
['.other.com', 'www.example.com']))
self.assert_(acl.referrer_allowed('http://www.example.com',
['-.example.com', 'www.example.com']))
# This is considered a relative uri to the request uri, a mode not
# currently supported.
self.assert_(not acl.referrer_allowed('www.example.com',
['.example.com']))
self.assert_(not acl.referrer_allowed('../index.html',
['.example.com']))
self.assert_(acl.referrer_allowed('www.example.com', ['*']))
if __name__ == '__main__':
unittest.main()

175
test/unit/common/middleware/test_auth.py
View File

@ -94,7 +94,7 @@ class Logger(object):
class FakeApp(object):
def __call__(self, env, start_response):
return "OK"
return ['204 No Content']
def start_response(*args):
pass
@ -102,75 +102,192 @@ def start_response(*args):
class TestAuth(unittest.TestCase):
def setUp(self):
self.test_auth = auth.DevAuthMiddleware(
FakeApp(), {}, FakeMemcache(), Logger())
self.test_auth = auth.filter_factory({})(FakeApp())
def test_auth_fail(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(404)
self.assertFalse(self.test_auth.auth('a','t'))
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()},
lambda x, y: None))
self.assert_(result.startswith('401'), result)
finally:
auth.http_connect = old_http_connect
def test_auth_success(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
self.assertTrue(self.test_auth.auth('a','t'))
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()},
lambda x, y: None))
self.assert_(result.startswith('204'), result)
finally:
auth.http_connect = old_http_connect
def test_auth_memcache(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
self.assertTrue(self.test_auth.auth('a','t'))
fake_memcache = FakeMemcache()
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
lambda x, y: None))
self.assert_(result.startswith('204'), result)
auth.http_connect = mock_http_connect(404)
# Should still be in memcache
self.assertTrue(self.test_auth.auth('a','t'))
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
lambda x, y: None))
self.assert_(result.startswith('204'), result)
finally:
auth.http_connect = old_http_connect
def test_auth_just_expired(self):
old_http_connect = auth.http_connect
try:
fake_memcache = FakeMemcache()
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
lambda x, y: None))
self.assert_(result.startswith('204'), result)
auth.http_connect = mock_http_connect(404)
# Should still be in memcache, but expired
result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
lambda x, y: None))
self.assert_(result.startswith('401'), result)
finally:
auth.http_connect = old_http_connect
def test_middleware_success(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
req = Request.blank('/v/a/c/o', headers={'x-auth-token':'t'})
resp = self.test_auth(req.environ, start_response)
self.assertEquals(resp, 'OK')
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
req = Request.blank('/v/a/c/o', headers={'x-auth-token': 'AUTH_t'})
req.environ['swift.cache'] = FakeMemcache()
result = ''.join(self.test_auth(req.environ, start_response))
self.assert_(result.startswith('204'), result)
self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa')
finally:
auth.http_connect = old_http_connect
def test_middleware_no_header(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
req = Request.blank('/v/a/c/o')
resp = self.test_auth(req.environ, start_response)
self.assertEquals(resp, ['Missing Auth Token'])
req.environ['swift.cache'] = FakeMemcache()
result = ''.join(self.test_auth(req.environ, start_response))
self.assert_(result.startswith('204'), result)
self.assert_(not req.remote_user, req.remote_user)
finally:
auth.http_connect = old_http_connect
def test_middleware_storage_token(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
req = Request.blank('/v/a/c/o', headers={'x-storage-token':'t'})
resp = self.test_auth(req.environ, start_response)
self.assertEquals(resp, 'OK')
auth.http_connect = mock_http_connect(204,
{'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
req = Request.blank('/v/a/c/o',
headers={'x-storage-token': 'AUTH_t'})
req.environ['swift.cache'] = FakeMemcache()
result = ''.join(self.test_auth(req.environ, start_response))
self.assert_(result.startswith('204'), result)
self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa')
finally:
auth.http_connect = old_http_connect
def test_middleware_only_version(self):
old_http_connect = auth.http_connect
try:
auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
req = Request.blank('/v', headers={'x-auth-token':'t'})
resp = self.test_auth(req.environ, start_response)
self.assertEquals(resp, ['Bad URL'])
finally:
auth.http_connect = old_http_connect
def test_authorize_bad_path(self):
req = Request.blank('/badpath')
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('401'), resp)
req = Request.blank('/badpath')
req.remote_user = 'act:usr,act,AUTH_cfa'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
def test_authorize_account_access(self):
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act,AUTH_cfa'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
def test_authorize_acl_group_access(self):
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = 'act'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = 'act:usr'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = 'act2'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = 'act:usr2'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
def test_deny_cross_reseller(self):
# Tests that cross-reseller is denied, even if ACLs/group names match
req = Request.blank('/v1/OTHER_cfa')
req.remote_user = 'act:usr,act,AUTH_cfa'
req.acl = 'act'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
def test_authorize_acl_referrer_access(self):
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = '.r:*'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.acl = '.r:.example.com'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('403'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.remote_user = 'act:usr,act'
req.referer = 'http://www.example.com/index.html'
req.acl = '.r:.example.com'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('401'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.acl = '.r:*'
self.assertEquals(self.test_auth.authorize(req), None)
req = Request.blank('/v1/AUTH_cfa')
req.acl = '.r:.example.com'
resp = str(self.test_auth.authorize(req))
self.assert_(resp.startswith('401'), resp)
req = Request.blank('/v1/AUTH_cfa')
req.referer = 'http://www.example.com/index.html'
req.acl = '.r:.example.com'
self.assertEquals(self.test_auth.authorize(req), None)
if __name__ == '__main__':

9
test/unit/common/test_client.py
View File

@ -191,7 +191,7 @@ class TestGetAccount(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_account('http://www.test.com', 'asdf')
value = c.get_account('http://www.test.com', 'asdf')[1]
self.assertEquals(value, [])
@ -200,7 +200,10 @@ class TestHeadAccount(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.head_account('http://www.tests.com', 'asdf')
self.assertEquals(value, (0, 0, 0))
# TODO: Hmm. This doesn't really test too much as it uses a fake that
# always returns the same dict. I guess it "exercises" the code, so
# I'll leave it for now.
self.assertEquals(type(value), dict)
def test_server_error(self):
c.http_connection = self.fake_http_connection(500)
@ -212,7 +215,7 @@ class TestGetContainer(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_container('http://www.test.com', 'asdf', 'asdf')
value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1]
self.assertEquals(value, [])

1
test/unit/common/test_direct_client.py
View File

@ -16,7 +16,6 @@
# TODO: Tests
import unittest
from swift.common import direct_client
class TestAuditor(unittest.TestCase):

43
test/unit/container/test_server.py
View File

@ -55,6 +55,49 @@ class TestContainerController(unittest.TestCase):
""" Tear down for testing swift.object_server.ObjectController """
rmtree(self.testdir, ignore_errors=1)
def test_acl_container(self):
# Ensure no acl by default
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': '0'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
response = self.controller.HEAD(req)
self.assert_(response.status.startswith('204'))
self.assert_('x-container-read' not in response.headers)
self.assert_('x-container-write' not in response.headers)
# Ensure POSTing acls works
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': '1', 'X-Container-Read': '.r:*',
'X-Container-Write': 'account:user'})
self.controller.POST(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
response = self.controller.HEAD(req)
self.assert_(response.status.startswith('204'))
self.assertEquals(response.headers.get('x-container-read'), '.r:*')
self.assertEquals(response.headers.get('x-container-write'),
'account:user')
# Ensure we can clear acls on POST
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': '3', 'X-Container-Read': '',
'X-Container-Write': ''})
self.controller.POST(req)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
response = self.controller.HEAD(req)
self.assert_(response.status.startswith('204'))
self.assert_('x-container-read' not in response.headers)
self.assert_('x-container-write' not in response.headers)
# Ensure PUTing acls works
req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': '4', 'X-Container-Read': '.r:*',
'X-Container-Write': 'account:user'})
self.controller.PUT(req)
req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'})
response = self.controller.HEAD(req)
self.assert_(response.status.startswith('204'))
self.assertEquals(response.headers.get('x-container-read'), '.r:*')
self.assertEquals(response.headers.get('x-container-write'),
'account:user')
def test_HEAD(self):
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})

43
test/unit/obj/test_server.py
View File

@ -19,6 +19,7 @@ import cPickle as pickle
import os
import sys
import unittest
from nose import SkipTest
from shutil import rmtree
from StringIO import StringIO
from time import gmtime, sleep, strftime, time
@ -64,7 +65,7 @@ class TestObjectController(unittest.TestCase):
def test_POST_update_meta(self):
""" Test swift.object_server.ObjectController.POST """
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -92,7 +93,7 @@ class TestObjectController(unittest.TestCase):
def test_POST_not_exist(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Timestamp': timestamp,
@ -114,7 +115,7 @@ class TestObjectController(unittest.TestCase):
def test_POST_container_connection(self):
if not self.path_to_test_xfs:
return
raise SkipTest
def mock_http_connect(response, with_exc=False):
class FakeConn(object):
def __init__(self, status, with_exc):
@ -210,7 +211,7 @@ class TestObjectController(unittest.TestCase):
def test_PUT_common(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -234,7 +235,7 @@ class TestObjectController(unittest.TestCase):
def test_PUT_overwrite(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '6',
@ -267,7 +268,7 @@ class TestObjectController(unittest.TestCase):
def test_PUT_no_etag(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Type': 'text/plain'})
@ -286,7 +287,7 @@ class TestObjectController(unittest.TestCase):
def test_PUT_user_metadata(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': timestamp,
@ -314,7 +315,7 @@ class TestObjectController(unittest.TestCase):
def test_PUT_container_connection(self):
if not self.path_to_test_xfs:
return
raise SkipTest
def mock_http_connect(response, with_exc=False):
class FakeConn(object):
def __init__(self, status, with_exc):
@ -376,7 +377,7 @@ class TestObjectController(unittest.TestCase):
def test_HEAD(self):
""" Test swift.object_server.ObjectController.HEAD """
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c')
resp = self.object_controller.HEAD(req)
self.assertEquals(resp.status_int, 400)
@ -443,7 +444,7 @@ class TestObjectController(unittest.TestCase):
def test_GET(self):
""" Test swift.object_server.ObjectController.GET """
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c')
resp = self.object_controller.GET(req)
self.assertEquals(resp.status_int, 400)
@ -532,7 +533,7 @@ class TestObjectController(unittest.TestCase):
def test_GET_if_match(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': normalize_timestamp(time()),
@ -586,7 +587,7 @@ class TestObjectController(unittest.TestCase):
def test_GET_if_none_match(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
'X-Timestamp': normalize_timestamp(time()),
@ -637,7 +638,7 @@ class TestObjectController(unittest.TestCase):
def test_GET_if_modified_since(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
@ -674,7 +675,7 @@ class TestObjectController(unittest.TestCase):
def test_GET_if_unmodified_since(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={
@ -713,7 +714,7 @@ class TestObjectController(unittest.TestCase):
def test_DELETE(self):
""" Test swift.object_server.ObjectController.DELETE """
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'})
resp = self.object_controller.DELETE(req)
self.assertEquals(resp.status_int, 400)
@ -840,7 +841,7 @@ class TestObjectController(unittest.TestCase):
def test_chunked_put(self):
if not self.path_to_test_xfs:
return
raise SkipTest
listener = listen(('localhost', 0))
port = listener.getsockname()[1]
killer = spawn(wsgi.server, listener, self.object_controller,
@ -866,7 +867,7 @@ class TestObjectController(unittest.TestCase):
def test_max_object_name_length(self):
if not self.path_to_test_xfs:
return
raise SkipTest
timestamp = normalize_timestamp(time())
req = Request.blank('/sda1/p/a/c/' + ('1' * 1024),
environ={'REQUEST_METHOD': 'PUT'},
@ -887,7 +888,7 @@ class TestObjectController(unittest.TestCase):
def test_disk_file_app_iter_corners(self):
if not self.path_to_test_xfs:
return
raise SkipTest
df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o')
mkdirs(df.datadir)
f = open(os.path.join(df.datadir,
@ -920,7 +921,7 @@ class TestObjectController(unittest.TestCase):
def test_max_upload_time(self):
if not self.path_to_test_xfs:
return
raise SkipTest
class SlowBody():
def __init__(self):
self.sent = 0
@ -962,7 +963,7 @@ class TestObjectController(unittest.TestCase):
def test_bad_sinces(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '4', 'Content-Type': 'text/plain'},
@ -988,7 +989,7 @@ class TestObjectController(unittest.TestCase):
def test_content_encoding(self):
if not self.path_to_test_xfs:
return
raise SkipTest
req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': normalize_timestamp(time()),
'Content-Length': '4', 'Content-Type': 'text/plain',

367
test/unit/proxy/test_server.py
View File

@ -19,6 +19,7 @@ import logging
import os
import sys
import unittest
from nose import SkipTest
from ConfigParser import ConfigParser
from contextlib import contextmanager
from cStringIO import StringIO
@ -33,6 +34,7 @@ from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen
from eventlet.timeout import Timeout
import simplejson
from webob import Request
from webob.exc import HTTPUnauthorized
from test.unit import connect_tcp, readuntil2crlfs
from swift.proxy import server as proxy_server
@ -81,7 +83,7 @@ def fake_http_connect(*code_iter, **kwargs):
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
return headers
return headers.items()
def read(self, amt=None):
if 'slow' in kwargs:
if self.sent < 4:
@ -97,7 +99,7 @@ def fake_http_connect(*code_iter, **kwargs):
self.received += 1
sleep(0.1)
def getheader(self, name, default=None):
return self.getheaders().get(name.lower(), default)
return dict(self.getheaders()).get(name.lower(), default)
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):
@ -203,10 +205,39 @@ class TestProxyServer(unittest.TestCase):
app = MyApp(None, FakeMemcache(), account_ring=FakeRing(),
container_ring=FakeRing(), object_ring=FakeRing())
req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
req.account = 'account'
app.update_request(req)
resp = app.handle_request(req)
self.assertEquals(resp.status_int, 500)
def test_calls_authorize_allow(self):
called = [False]
def authorize(req):
called[0] = True
with save_globals():
proxy_server.http_connect = fake_http_connect(200)
app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize
app.update_request(req)
resp = app.handle_request(req)
self.assert_(called[0])
def test_calls_authorize_deny(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
app = proxy_server.Application(None, FakeMemcache(),
account_ring=FakeRing(), container_ring=FakeRing(),
object_ring=FakeRing())
req = Request.blank('/v1/a')
req.environ['swift.authorize'] = authorize
app.update_request(req)
resp = app.handle_request(req)
self.assert_(called[0])
class TestObjectController(unittest.TestCase):
@ -224,14 +255,14 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
@ -244,7 +275,7 @@ class TestObjectController(unittest.TestCase):
give_content_type=lambda content_type:
self.assertEquals(content_type, expected.next()))
req = Request.blank('/a/c/%s' % filename, {})
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
test_content_type('test.jpg',
iter(['', '', '', 'image/jpeg', 'image/jpeg', 'image/jpeg']))
@ -261,7 +292,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
req.account = 'a'
self.app.update_request(req)
self.app.memcache.store = {}
res = controller.PUT(req)
expected = str(expected)
@ -296,7 +327,7 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {}
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
@ -330,7 +361,7 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {}
proxy_server.http_connect = mock_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', {})
req.account = 'a'
self.app.update_request(req)
req.body_file = StringIO('some data')
res = controller.PUT(req)
expected = str(expected)
@ -347,7 +378,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', {}, headers={
'Content-Length': str(MAX_FILE_SIZE + 1),
'Content-Type': 'foo/bar'})
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
self.assertEquals(res.status_int, 413)
@ -379,7 +410,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = mock_http_connect(*statuses)
req = Request.blank('/a/c/o.jpg', {})
req.content_length = 0
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(str(expected))],
@ -397,7 +428,7 @@ class TestObjectController(unittest.TestCase):
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar'})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
@ -417,7 +448,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
req.account = 'a'
self.app.update_request(req)
res = controller.DELETE(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
@ -436,7 +467,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(*statuses)
self.app.memcache.store = {}
req = Request.blank('/a/c/o', {})
req.account = 'a'
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
@ -460,14 +491,14 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x'*256})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
'X-Object-Meta-Foo': 'x'*257})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
@ -481,14 +512,14 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-'+'x'*128): 'x'})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 202)
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers={
'Content-Type': 'foo/bar',
('X-Object-Meta-'+'x'*129): 'x'})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
@ -500,7 +531,7 @@ class TestObjectController(unittest.TestCase):
headers.update({'Content-Type': 'foo/bar'})
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
@ -512,7 +543,7 @@ class TestObjectController(unittest.TestCase):
headers.update({'Content-Type': 'foo/bar'})
proxy_server.http_connect = fake_http_connect(202, 202, 202)
req = Request.blank('/a/c/o', {}, headers=headers)
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(res.status_int, 400)
@ -542,7 +573,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
@ -554,7 +585,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
req.account = 'account'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(201, 201, 201)
# obj obj obj
@ -583,7 +614,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o',
environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
@ -607,7 +638,7 @@ class TestObjectController(unittest.TestCase):
dev['ip'] = '127.0.0.1'
dev['port'] = 1
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
@ -649,7 +680,7 @@ class TestObjectController(unittest.TestCase):
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
body=' ')
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = \
@ -663,7 +694,7 @@ class TestObjectController(unittest.TestCase):
environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
body=' ')
req.account = 'account'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 503)
@ -708,7 +739,7 @@ class TestObjectController(unittest.TestCase):
def test_proxy_passes_content_type(self):
with save_globals():
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = fake_http_connect(200, 200, 200)
@ -728,7 +759,7 @@ class TestObjectController(unittest.TestCase):
def test_proxy_passes_content_length_on_head(self):
with save_globals():
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
req.account = 'account'
self.app.update_request(req)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
proxy_server.http_connect = fake_http_connect(200, 200, 200)
@ -777,7 +808,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
req.account = 'a'
self.app.update_request(req)
resp = getattr(controller, 'DELETE')(req)
self.assertEquals(resp.status_int, 200)
@ -853,7 +884,7 @@ class TestObjectController(unittest.TestCase):
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 404)
@ -861,7 +892,7 @@ class TestObjectController(unittest.TestCase):
fake_http_connect(404, 404, 404, 200, 200, 200)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Type': 'text/plain'})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 404)
@ -874,7 +905,7 @@ class TestObjectController(unittest.TestCase):
# acct cont obj obj obj
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
@ -883,7 +914,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Object-Meta-' + ('a' *
MAX_META_NAME_LENGTH) : 'v'})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -891,7 +922,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Object-Meta-' + ('a' *
(MAX_META_NAME_LENGTH + 1)) : 'v'})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
@ -900,7 +931,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
MAX_META_VALUE_LENGTH})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -908,7 +939,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Object-Meta-Too-Long': 'a' *
(MAX_META_VALUE_LENGTH + 1)})
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
@ -918,7 +949,7 @@ class TestObjectController(unittest.TestCase):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -927,7 +958,7 @@ class TestObjectController(unittest.TestCase):
headers['X-Object-Meta-%d' % x] = 'v'
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
@ -946,7 +977,7 @@ class TestObjectController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -954,7 +985,7 @@ class TestObjectController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size)
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.PUT(req)
self.assertEquals(resp.status_int, 400)
@ -964,7 +995,7 @@ class TestObjectController(unittest.TestCase):
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
# acct cont obj obj obj
@ -974,7 +1005,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': 'c/o'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
# acct cont acct cont objc obj obj obj
@ -986,7 +1017,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
# acct cont acct cont objc obj obj obj
@ -998,7 +1029,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 503, 503, 503)
# acct cont objc objc objc
@ -1009,7 +1040,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 404, 404, 404)
# acct cont objc objc objc
@ -1020,7 +1051,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0',
'X-Copy-From': '/c/o'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201)
# acct cont objc objc objc obj obj obj
@ -1032,7 +1063,7 @@ class TestObjectController(unittest.TestCase):
headers={'Content-Length': '0',
'X-Copy-From': '/c/o',
'X-Object-Meta-Ours': 'okay'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = \
fake_http_connect(200, 200, 200, 201, 201, 201)
# acct cont objc obj obj obj
@ -1052,7 +1083,7 @@ class TestObjectController(unittest.TestCase):
'pointing to a valid directory.\n' \
'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
'system for testing.'
return
raise SkipTest
testdir = \
os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
mkdirs(testdir)
@ -1434,7 +1465,7 @@ class TestObjectController(unittest.TestCase):
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '0'})
req.account = 'a'
self.app.update_request(req)
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201,
etags=[None,
'68b329da9893e34099c7d8ad5cb9c940',
@ -1452,7 +1483,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '10'},
body='1234567890')
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
self.assert_(hasattr(req, 'bytes_transferred'))
self.assertEquals(req.bytes_transferred, 10)
@ -1464,7 +1495,7 @@ class TestObjectController(unittest.TestCase):
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o')
req.account = 'a'
self.app.update_request(req)
res = controller.GET(req)
res.body
self.assert_(hasattr(res, 'bytes_transferred'))
@ -1479,7 +1510,7 @@ class TestObjectController(unittest.TestCase):
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '10'},
body='12345')
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
self.assertEquals(req.bytes_transferred, 5)
self.assert_(hasattr(req, 'client_disconnect'))
@ -1492,7 +1523,7 @@ class TestObjectController(unittest.TestCase):
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o')
req.account = 'a'
self.app.update_request(req)
orig_object_chunk_size = self.app.object_chunk_size
try:
self.app.object_chunk_size = 5
@ -1509,6 +1540,73 @@ class TestObjectController(unittest.TestCase):
finally:
self.app.object_chunk_size = orig_object_chunk_size
def test_GET_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.GET(req)
self.assert_(called[0])
def test_HEAD_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', {'REQUEST_METHOD': 'HEAD'})
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.HEAD(req)
self.assert_(called[0])
def test_POST_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
headers={'Content-Length': '5'}, body='12345')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.POST(req)
self.assert_(called[0])
def test_PUT_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 200, 201, 201, 201)
controller = proxy_server.ObjectController(self.app, 'account',
'container', 'object')
req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
headers={'Content-Length': '5'}, body='12345')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.PUT(req)
self.assert_(called[0])
class TestContainerController(unittest.TestCase):
"Test swift.proxy_server.ContainerController"
@ -1528,14 +1626,14 @@ class TestContainerController(unittest.TestCase):
self.app.memcache.store = {}
req = Request.blank('/a/c', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c/', headers={'Content-Length': '0',
'Content-Type': 'text/plain'})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
@ -1547,7 +1645,7 @@ class TestContainerController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
self.app.memcache.store = {}
req = Request.blank('/a/c', {})
req.account = 'a'
self.app.update_request(req)
res = controller.HEAD(req)
self.assertEquals(res.status[:len(str(expected))],
str(expected))
@ -1570,7 +1668,7 @@ class TestContainerController(unittest.TestCase):
self.app.memcache.store = {}
req = Request.blank('/a/c', {})
req.content_length = 0
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
expected = str(expected)
self.assertEquals(res.status[:len(expected)], expected)
@ -1613,7 +1711,7 @@ class TestContainerController(unittest.TestCase):
fake_http_connect(200, 200, 200, 200)
self.app.memcache.store = {}
req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
req.account = 'a'
self.app.update_request(req)
resp = getattr(controller, meth)(req)
self.assertEquals(resp.status_int, 200)
@ -1657,7 +1755,7 @@ class TestContainerController(unittest.TestCase):
self.app.memcache = MockMemcache(allow_lock=True)
proxy_server.http_connect = fake_http_connect(200, 200, 200, 201, 201, 201, missing_container=True)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'})
req.account = 'a'
self.app.update_request(req)
res = controller.PUT(req)
self.assertEquals(res.status_int, 201)
@ -1703,7 +1801,7 @@ class TestContainerController(unittest.TestCase):
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c?format=json')
req.account = 'a'
self.app.update_request(req)
res = controller.GET(req)
res.body
self.assert_(hasattr(res, 'bytes_transferred'))
@ -1715,7 +1813,7 @@ class TestContainerController(unittest.TestCase):
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c?format=json')
req.account = 'a'
self.app.update_request(req)
orig_object_chunk_size = self.app.object_chunk_size
try:
self.app.object_chunk_size = 1
@ -1760,8 +1858,7 @@ class TestContainerController(unittest.TestCase):
201, give_connect=test_connect)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers={test_header: test_value})
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
res = getattr(controller, method)(req)
self.assertEquals(test_errors, [])
@ -1776,7 +1873,7 @@ class TestContainerController(unittest.TestCase):
controller = proxy_server.ContainerController(self.app, 'a', 'c')
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method})
req.account = 'a'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 201)
@ -1784,16 +1881,14 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers={'X-Container-Meta-' +
('a' * MAX_META_NAME_LENGTH): 'v'})
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers={'X-Container-Meta-' +
('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 400)
@ -1801,16 +1896,14 @@ class TestContainerController(unittest.TestCase):
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers={'X-Container-Meta-Too-Long':
'a' * MAX_META_VALUE_LENGTH})
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers={'X-Container-Meta-Too-Long':
'a' * (MAX_META_VALUE_LENGTH + 1)})
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 400)
@ -1820,8 +1913,7 @@ class TestContainerController(unittest.TestCase):
headers['X-Container-Meta-%d' % x] = 'v'
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers=headers)
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -1830,8 +1922,7 @@ class TestContainerController(unittest.TestCase):
headers['X-Container-Meta-%d' % x] = 'v'
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers=headers)
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 400)
@ -1849,8 +1940,7 @@ class TestContainerController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers=headers)
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 201)
proxy_server.http_connect = fake_http_connect(201, 201, 201)
@ -1858,11 +1948,96 @@ class TestContainerController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size)
req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
headers=headers)
req.account = 'a'
req.container = 'c'
self.app.update_request(req)
resp = getattr(controller, method)(req)
self.assertEquals(resp.status_int, 400)
def test_POST_calls_clean_acl(self):
called = [False]
def clean_acl(header, value):
called[0] = True
raise ValueError('fake error')
with save_globals():
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Container-Read': '.r:*'})
req.environ['swift.clean_acl'] = clean_acl
self.app.update_request(req)
res = controller.POST(req)
self.assert_(called[0])
called[0] = False
with save_globals():
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Container-Write': '.r:*'})
req.environ['swift.clean_acl'] = clean_acl
self.app.update_request(req)
res = controller.POST(req)
self.assert_(called[0])
def test_PUT_calls_clean_acl(self):
called = [False]
def clean_acl(header, value):
called[0] = True
raise ValueError('fake error')
with save_globals():
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Container-Read': '.r:*'})
req.environ['swift.clean_acl'] = clean_acl
self.app.update_request(req)
res = controller.PUT(req)
self.assert_(called[0])
called[0] = False
with save_globals():
proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Container-Write': '.r:*'})
req.environ['swift.clean_acl'] = clean_acl
self.app.update_request(req)
res = controller.PUT(req)
self.assert_(called[0])
def test_GET_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c')
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.GET(req)
self.assert_(called[0])
def test_HEAD_calls_authorize(self):
called = [False]
def authorize(req):
called[0] = True
return HTTPUnauthorized(request=req)
with save_globals():
proxy_server.http_connect = \
fake_http_connect(200, 201, 201, 201)
controller = proxy_server.ContainerController(self.app, 'account',
'container')
req = Request.blank('/a/c', {'REQUEST_METHOD': 'HEAD'})
req.environ['swift.authorize'] = authorize
self.app.update_request(req)
res = controller.HEAD(req)
self.assert_(called[0])
class TestAccountController(unittest.TestCase):
@ -1875,12 +2050,12 @@ class TestAccountController(unittest.TestCase):
with save_globals():
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a', {})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
proxy_server.http_connect = fake_http_connect(*statuses)
req = Request.blank('/a/', {})
req.account = 'a'
self.app.update_request(req)
res = method(req)
self.assertEquals(res.status_int, expected)
@ -1930,7 +2105,7 @@ class TestAccountController(unittest.TestCase):
dev['port'] = 1 ## can't connect on this port
controller = proxy_server.AccountController(self.app, 'account')
req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
req.account = 'account'
self.app.update_request(req)
resp = controller.HEAD(req)
self.assertEquals(resp.status_int, 503)
@ -1941,7 +2116,7 @@ class TestAccountController(unittest.TestCase):
dev['port'] = -1 ## invalid port number
controller = proxy_server.AccountController(self.app, 'account')
req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
req.account = 'account'
self.app.update_request(req)
resp = controller.HEAD(req)
self.assertEquals(resp.status_int, 503)
@ -1950,7 +2125,7 @@ class TestAccountController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(200, 200, body='{}')
controller = proxy_server.AccountController(self.app, 'account')
req = Request.blank('/a?format=json')
req.account = 'a'
self.app.update_request(req)
res = controller.GET(req)
res.body
self.assert_(hasattr(res, 'bytes_transferred'))
@ -1961,7 +2136,7 @@ class TestAccountController(unittest.TestCase):
proxy_server.http_connect = fake_http_connect(200, 200, body='{}')
controller = proxy_server.AccountController(self.app, 'account')
req = Request.blank('/a?format=json')
req.account = 'a'
self.app.update_request(req)
orig_object_chunk_size = self.app.object_chunk_size
try:
self.app.object_chunk_size = 1
@ -1999,7 +2174,7 @@ class TestAccountController(unittest.TestCase):
give_connect=test_connect)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers={test_header: test_value})
req.account = 'a'
self.app.update_request(req)
res = controller.POST(req)
self.assertEquals(test_errors, [])
@ -2008,7 +2183,7 @@ class TestAccountController(unittest.TestCase):
controller = proxy_server.AccountController(self.app, 'a')
proxy_server.http_connect = fake_http_connect(204, 204, 204)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 204)
@ -2016,14 +2191,14 @@ class TestAccountController(unittest.TestCase):
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Account-Meta-' +
('a' * MAX_META_NAME_LENGTH): 'v'})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 204)
proxy_server.http_connect = fake_http_connect(204, 204, 204)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Account-Meta-' +
('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 400)
@ -2031,14 +2206,14 @@ class TestAccountController(unittest.TestCase):
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Account-Meta-Too-Long':
'a' * MAX_META_VALUE_LENGTH})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 204)
proxy_server.http_connect = fake_http_connect(204, 204, 204)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers={'X-Account-Meta-Too-Long':
'a' * (MAX_META_VALUE_LENGTH + 1)})
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 400)
@ -2048,7 +2223,7 @@ class TestAccountController(unittest.TestCase):
headers['X-Account-Meta-%d' % x] = 'v'
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 204)
proxy_server.http_connect = fake_http_connect(204, 204, 204)
@ -2057,7 +2232,7 @@ class TestAccountController(unittest.TestCase):
headers['X-Account-Meta-%d' % x] = 'v'
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 400)
@ -2075,7 +2250,7 @@ class TestAccountController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size - 1)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 204)
proxy_server.http_connect = fake_http_connect(204, 204, 204)
@ -2083,7 +2258,7 @@ class TestAccountController(unittest.TestCase):
'a' * (MAX_META_OVERALL_SIZE - size)
req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
headers=headers)
req.account = 'a'
self.app.update_request(req)
resp = controller.POST(req)
self.assertEquals(resp.status_int, 400)