openstack/swift
Code Issues Proposed changes

Updated tools and client.py to work with ACLs

Browse Source
This commit is contained in:
gholt 2010-09-03 21:39:44 -07:00
parent 28ac96b90d
commit bb01c22440
10 changed files with 609 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

450
bin/st
View File

@ -154,19 +154,26 @@ except:
def get_auth(url, user, key, snet=False):
"""
Get authentication credentials
Get authentication/authorization credentials.
:param url: authentication URL
:param user: user to auth as
:param key: key or passowrd for auth
:param snet: use SERVICENET internal network default is False
:returns: tuple of (storage URL, storage token, auth token)
The snet parameter is used for Rackspace's ServiceNet internal network
implementation. In this function, it simply adds *snet-* to the beginning
of the host name for the returned storage URL. With Rackspace Cloud Files,
use of this network path causes no bandwidth charges but requires the
client to be running on Rackspace's ServiceNet network.
:param url: authentication/authorization URL
:param user: user to authenticate as
:param key: key or password for authorization
:param snet: use SERVICENET internal network (see above), default is False
:returns: tuple of (storage URL, auth token)
:raises ClientException: HTTP GET request to auth URL failed
"""
parsed, conn = http_connection(url)
conn.request('GET', parsed.path, '',
{'X-Auth-User': user, 'X-Auth-Key': key})
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
@ -175,7 +182,7 @@ except:
url = resp.getheader('x-storage-url')
if snet:
parsed = list(urlparse(url))
# Second item in the list is the netloc
# Second item in the list is the netloc
parsed[1] = 'snet-' + parsed[1]
url = urlunparse(parsed)
return url, resp.getheader('x-storage-token',
@ -196,18 +203,21 @@ except:
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of accounts
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_account(url, token, marker, limit, prefix, http_conn)
rv = get_account(url, token, marker, limit, prefix, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
marker = listing[-1]['name']
listing = get_account(url, token, marker, limit, prefix, http_conn)
listing = \
get_account(url, token, marker, limit, prefix, http_conn)[1]
if listing:
rv.extend(listing)
return rv
parsed, conn = http_conn
qs = 'format=json'
@ -220,6 +230,9 @@ except:
conn.request('GET', '%s?%s' % (parsed.path, qs), '',
{'X-Auth-Token': token})
resp = conn.getresponse()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@ -228,8 +241,8 @@ except:
http_reason=resp.reason)
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_account(url, token, http_conn=None):
@ -240,7 +253,8 @@ except:
:param token: auth token
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (container count, object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -249,14 +263,42 @@ except:
parsed, conn = http_connection(url)
conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-account-container-count', 0)), \
int(resp.getheader('x-account-object-count', 0)), \
int(resp.getheader('x-account-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def post_account(url, token, headers, http_conn=None):
"""
Update an account's metadata.
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
headers['X-Auth-Token'] = token
conn.request('POST', parsed.path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def get_container(url, token, container, marker=None, limit=None,
@ -276,23 +318,25 @@ except:
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, http_conn)
prefix, delimiter, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
path = '%s/%s' % (parsed.path, quote(container))
@ -313,10 +357,13 @@ except:
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_query=qs,
http_status=resp.status, http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None):
@ -328,7 +375,8 @@ except:
:param container: container name to get stats for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -344,17 +392,20 @@ except:
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count', 0)), \
int(resp.getheader('x-container-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_container(url, token, container, http_conn=None):
def put_container(url, token, container, headers=None, http_conn=None):
"""
Create a container
:param url: storage URL
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP PUT request failed
@ -364,7 +415,10 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('PUT', path, '', {'X-Auth-Token': token})
if not headers:
headers = {}
headers['X-Auth-Token'] = token
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
@ -374,6 +428,34 @@ except:
http_reason=resp.reason)
def post_container(url, token, container, headers, http_conn=None):
"""
Update a container's metadata.
:param url: storage URL
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def delete_container(url, token, container, http_conn=None):
"""
Delete a container
@ -411,8 +493,12 @@ except:
:param name: object name to get
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param resp_chunk_size: if defined, chunk size of data to read
:returns: a list of objects
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
request.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if http_conn:
@ -441,12 +527,10 @@ except:
object_body = _object_body()
else:
object_body = resp.read()
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata, \
object_body
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def head_object(url, token, container, name, http_conn=None):
@ -459,8 +543,8 @@ except:
:param name: object name to get info for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (content type, content length, last modfied, etag,
dictionary of metadata)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -479,16 +563,15 @@ except:
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_object(url, token, container, name, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None, http_conn=None):
def put_object(url, token, container, name, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None, headers=None,
http_conn=None):
"""
Put an object
@ -496,12 +579,12 @@ except:
:param token: auth token
:param container: container name that the object is in
:param name: object name to put
:param contents: file like object to read object data from
:param metadata: dictionary of object metadata
:param contents: a string or a file like object to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param chunk_size: chunk size of data to write
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: etag from server response
@ -512,9 +595,9 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
if not headers:
headers = {}
headers['X-Auth-Token'] = token
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
@ -550,7 +633,7 @@ except:
return resp.getheader('etag').strip('"')
def post_object(url, token, container, name, metadata, http_conn=None):
def post_object(url, token, container, name, headers, http_conn=None):
"""
Change object metadata
@ -558,7 +641,7 @@ except:
:param token: auth token
:param container: container name that the object is in
:param name: object name to change
:param metadata: dictionary of object metadata
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
@ -568,9 +651,7 @@ except:
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
@ -620,7 +701,7 @@ except:
:param preauthurl: storage URL (if you have already authenticated)
:param preauthtoken: authentication token (if you have already
authenticated)
:param snet: use SERVICENET internal network default is False
:param snet: use SERVICENET internal network default is False
"""
self.authurl = authurl
self.user = user
@ -632,20 +713,24 @@ except:
self.attempts = 0
self.snet = snet
def get_auth(self):
return get_auth(self.authurl, self.user, self.key, snet=self.snet)
def http_connection(self):
return http_connection(self.url)
def _retry(self, func, *args, **kwargs):
kwargs['http_conn'] = self.http_conn
self.attempts = 0
backoff = 1
while self.attempts <= self.retries:
self.attempts += 1
try:
if not self.url or not self.token:
self.url, self.token = \
get_auth(self.authurl, self.user, self.key, snet=self.snet)
self.url, self.token = self.get_auth()
self.http_conn = None
if not self.http_conn:
self.http_conn = http_connection(self.url)
kwargs['http_conn'] = self.http_conn
self.http_conn = self.http_connection()
kwargs['http_conn'] = self.http_conn
rv = func(self.url, self.token, *args, **kwargs)
return rv
except (socket.error, HTTPException):
@ -667,63 +752,71 @@ except:
backoff *= 2
def head_account(self):
"""Wrapper for head_account"""
"""Wrapper for :func:`head_account`"""
return self._retry(head_account)
def get_account(self, marker=None, limit=None, prefix=None,
full_listing=False):
"""Wrapper for get_account"""
# TODO: With full_listing=True this will restart the entire listing
# with each retry. Need to make a better version that just retries
# where it left off.
"""Wrapper for :func:`get_account`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(get_account, marker=marker, limit=limit,
prefix=prefix, full_listing=full_listing)
def post_account(self, headers):
"""Wrapper for :func:`post_account`"""
return self._retry(post_account, headers)
def head_container(self, container):
"""Wrapper for head_container"""
"""Wrapper for :func:`head_container`"""
return self._retry(head_container, container)
def get_container(self, container, marker=None, limit=None, prefix=None,
delimiter=None, full_listing=False):
"""Wrapper for get_container"""
# TODO: With full_listing=True this will restart the entire listing
# with each retry. Need to make a better version that just retries
# where it left off.
"""Wrapper for :func:`get_container`"""
# TODO(unknown): With full_listing=True this will restart the entire
# listing with each retry. Need to make a better version that just
# retries where it left off.
return self._retry(get_container, container, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
def put_container(self, container):
"""Wrapper for put_container"""
return self._retry(put_container, container)
def put_container(self, container, headers=None):
"""Wrapper for :func:`put_container`"""
return self._retry(put_container, container, headers=headers)
def post_container(self, container, headers):
"""Wrapper for :func:`post_container`"""
return self._retry(post_container, container, headers)
def delete_container(self, container):
"""Wrapper for delete_container"""
"""Wrapper for :func:`delete_container`"""
return self._retry(delete_container, container)
def head_object(self, container, obj):
"""Wrapper for head_object"""
"""Wrapper for :func:`head_object`"""
return self._retry(head_object, container, obj)
def get_object(self, container, obj, resp_chunk_size=None):
"""Wrapper for get_object"""
"""Wrapper for :func:`get_object`"""
return self._retry(get_object, container, obj,
resp_chunk_size=resp_chunk_size)
def put_object(self, container, obj, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None):
"""Wrapper for put_object"""
def put_object(self, container, obj, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None,
headers=None):
"""Wrapper for :func:`put_object`"""
return self._retry(put_object, container, obj, contents,
metadata=metadata, content_length=content_length, etag=etag,
chunk_size=chunk_size, content_type=content_type)
content_length=content_length, etag=etag, chunk_size=chunk_size,
content_type=content_type, headers=headers)
def post_object(self, container, obj, metadata):
"""Wrapper for post_object"""
return self._retry(post_object, container, obj, metadata)
def post_object(self, container, obj, headers):
"""Wrapper for :func:`post_object`"""
return self._retry(post_object, container, obj, headers)
def delete_object(self, container, obj):
"""Wrapper for delete_object"""
"""Wrapper for :func:`delete_object`"""
return self._retry(delete_object, container, obj)
# End inclusion of swift.common.client
@ -805,7 +898,7 @@ def st_delete(options, args):
marker = ''
while True:
objects = [o['name'] for o in
conn.get_container(container, marker=marker)]
conn.get_container(container, marker=marker)[1]]
if not objects:
break
for obj in objects:
@ -847,7 +940,7 @@ def st_delete(options, args):
marker = ''
while True:
containers = \
[c['name'] for c in conn.get_account(marker=marker)]
[c['name'] for c in conn.get_account(marker=marker)[1]]
if not containers:
break
for container in containers:
@ -893,8 +986,11 @@ def st_download(options, args):
object_queue = Queue(10000)
def _download_object((container, obj), conn):
try:
content_type, content_length, _, etag, metadata, body = \
headers, body = \
conn.get_object(container, obj, resp_chunk_size=65536)
content_type = headers.get('content-type')
content_length = int(headers.get('content-length'))
etag = headers.get('etag')
path = options.yes_all and join(container, obj) or obj
if path[:1] in ('/', '\\'):
path = path[1:]
@ -925,8 +1021,8 @@ def st_download(options, args):
options.error_queue.put(
'%s: read_length != content_length, %d != %d' %
(path, read_length, content_length))
if 'mtime' in metadata:
mtime = float(metadata['mtime'])
if 'x-object-meta-mtime' in headers:
mtime = float(headers['x-object-meta-mtime'])
utime(path, (mtime, mtime))
if options.verbose:
options.print_queue.put(path)
@ -941,7 +1037,7 @@ def st_download(options, args):
marker = ''
while True:
objects = [o['name'] for o in
conn.get_container(container, marker=marker)]
conn.get_container(container, marker=marker)[1]]
if not objects:
break
for obj in objects:
@ -969,7 +1065,7 @@ def st_download(options, args):
marker = ''
while True:
containers = [c['name']
for c in conn.get_account(marker=marker)]
for c in conn.get_account(marker=marker)[1]]
if not containers:
break
for container in containers:
@ -1016,10 +1112,11 @@ def st_list(options, args):
marker = ''
while True:
if not args:
items = conn.get_account(marker=marker, prefix=options.prefix)
items = \
conn.get_account(marker=marker, prefix=options.prefix)[1]
else:
items = conn.get_container(args[0], marker=marker,
prefix=options.prefix, delimiter=options.delimiter)
prefix=options.prefix, delimiter=options.delimiter)[1]
if not items:
break
for item in items:
@ -1042,46 +1139,85 @@ def st_stat(options, args):
conn = Connection(options.auth, options.user, options.key)
if not args:
try:
container_count, object_count, bytes_used = conn.head_account()
headers = conn.head_account()
container_count = int(headers.get('x-account-container-count', 0))
object_count = int(headers.get('x-account-object-count', 0))
bytes_used = int(headers.get('x-account-bytes-used', 0))
options.print_queue.put('''
Account: %s
Containers: %d
Objects: %d
Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count,
object_count, bytes_used))
for key, value in headers.items():
if key.startswith('x-account-meta-'):
options.print_queue.put('%10s: %s' % ('Meta %s' %
key[len('x-account-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-account-meta-') and key not in (
'content-length', 'date', 'x-account-container-count',
'x-account-object-count', 'x-account-bytes-used'):
options.print_queue.put(
'%10s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
elif len(args) == 1:
try:
object_count, bytes_used = conn.head_container(args[0])
headers = conn.head_container(args[0])
object_count = int(headers.get('x-container-object-count', 0))
bytes_used = int(headers.get('x-container-bytes-used', 0))
options.print_queue.put('''
Account: %s
Container: %s
Objects: %d
Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
object_count, bytes_used))
Bytes: %d
Read ACL: %s
Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
object_count, bytes_used,
headers.get('x-container-read', ''),
headers.get('x-container-write', '')))
for key, value in headers.items():
if key.startswith('x-container-meta-'):
options.print_queue.put('%9s: %s' % ('Meta %s' %
key[len('x-container-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-container-meta-') and key not in (
'content-length', 'date', 'x-container-object-count',
'x-container-bytes-used', 'x-container-read',
'x-container-write'):
options.print_queue.put(
'%9s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Container %s not found' % repr(args[0]))
elif len(args) == 2:
try:
content_type, content_length, last_modified, etag, metadata = \
conn.head_object(args[0], args[1])
headers = conn.head_object(args[0], args[1])
options.print_queue.put('''
Account: %s
Container: %s
Object: %s
Content Type: %s
Content Length: %d
Content Length: %s
Last Modified: %s
ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
args[1], content_type, content_length,
last_modified, etag))
for key, value in metadata.items():
options.print_queue.put('%14s: %s' % ('Meta %s' % key, value))
args[1], headers.get('content-type'),
headers.get('content-length'),
headers.get('last-modified'),
headers.get('etag')))
for key, value in headers.items():
if key.startswith('x-object-meta-'):
options.print_queue.put('%14s: %s' % ('Meta %s' %
key[len('x-object-meta-'):].title(), value))
for key, value in headers.items():
if not key.startswith('x-object-meta-') and key not in (
'content-type', 'content-length', 'last-modified',
'etag', 'date'):
options.print_queue.put(
'%14s: %s' % (key.title(), value))
except ClientException, err:
if err.http_status != 404:
raise
@ -1092,6 +1228,63 @@ Content Length: %d
(basename(argv[0]), st_stat_help))
st_post_help = '''
post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
or --meta option is allowed on all and used to define the user meta data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
def st_post(options, args):
conn = Connection(options.auth, options.user, options.key)
if not args:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Account-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
try:
conn.post_account(headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Account not found')
elif len(args) == 1:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Container-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
if options.read_acl is not None:
headers['X-Container-Read'] = options.read_acl
if options.write_acl is not None:
headers['X-Container-Write'] = options.write_acl
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
conn.put_container(args[0], headers=headers)
elif len(args) == 2:
headers = {}
for item in options.meta:
split_item = item.split(':')
headers['X-Object-Meta-' + split_item[0]] = \
len(split_item) > 1 and split_item[1]
try:
conn.post_object(args[0], args[1], headers=headers)
except ClientException, err:
if err.http_status != 404:
raise
options.error_queue.put('Object %s not found' %
repr('%s/%s' % (args[0], args[1])))
else:
options.error_queue.put('Usage: %s [options] %s' %
(basename(argv[0]), st_post_help))
st_upload_help = '''
upload [options] container file_or_directory [file_or_directory] [...]
Uploads to the given container the files and directories specified by the
@ -1108,35 +1301,41 @@ def st_upload(options, args):
obj = path
if obj.startswith('./') or obj.startswith('.\\'):
obj = obj[2:]
metadata = {'mtime': str(getmtime(path))}
put_headers = {'x-object-meta-mtime': str(getmtime(path))}
if dir_marker:
if options.changed:
try:
ct, cl, lm, et, md = conn.head_object(args[0], obj)
headers = conn.head_object(args[0], obj)
ct = headers.get('content-type')
cl = int(headers.get('content-length'))
et = headers.get('etag')
mt = headers.get('x-object-meta-mtime')
if ct.split(';', 1)[0] == 'text/directory' and \
cl == 0 and \
et == 'd41d8cd98f00b204e9800998ecf8427e' and \
md.get('mtime') == metadata['mtime']:
mt == put_headers['x-object-meta-mtime']:
return
except ClientException, err:
if err.http_status != 404:
raise
conn.put_object(args[0], obj, '', content_length=0,
content_type='text/directory',
metadata=metadata)
headers=put_headers)
else:
if options.changed:
try:
ct, cl, lm, et, md = conn.head_object(args[0], obj)
headers = conn.head_object(args[0], obj)
cl = int(headers.get('content-length'))
mt = headers.get('x-object-meta-mtime')
if cl == getsize(path) and \
md.get('mtime') == metadata['mtime']:
mt == put_headers['x-object-meta-mtime']:
return
except ClientException, err:
if err.http_status != 404:
raise
conn.put_object(args[0], obj, open(path, 'rb'),
content_length=getsize(path),
metadata=metadata)
headers=put_headers)
if options.verbose:
options.print_queue.put(obj)
except OSError, err:
@ -1190,6 +1389,7 @@ Commands:
%(st_stat_help)s
%(st_list_help)s
%(st_upload_help)s
%(st_post_help)s
%(st_download_help)s
%(st_delete_help)s
@ -1215,6 +1415,18 @@ Example:
help='For the list command on containers: will roll up '
'items with the given delimiter (see Cloud Files '
'general documentation for what this means).')
parser.add_option('-r', '--read-acl', dest='read_acl',
help='Sets the Read ACL with post container commands. '
'Quick summary of ACL syntax: .ref:any, '
'.ref:-.example.com, .ref:www.example.com, account1, '
'account2:user2')
parser.add_option('-w', '--write-acl', dest='write_acl',
help='Sets the Write ACL with post container commands. '
'Quick summary of ACL syntax: account1, account2:user2')
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item of the syntax name:value '
'for use with post commands. This option may be '
'repeated. Example: -m Color:Blue -m Size:Large')
parser.add_option('-A', '--auth', dest='auth',
help='URL for obtaining an auth token')
parser.add_option('-U', '--user', dest='user',
@ -1235,7 +1447,7 @@ overridden with -A, -U, or -K.'''.strip('\n')
if not getattr(options, attr, None):
exit(required_help)
commands = ('delete', 'download', 'list', 'stat', 'upload')
commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
if not args or args[0] not in commands:
parser.print_usage()
if args:

3
bin/swift-stats-populate
View File

@ -48,7 +48,8 @@ def put_object(connpool, container, obj, report):
global retries_done
try:
with connpool.item() as conn:
conn.put_object(container, obj, obj, metadata={'stats': obj})
conn.put_object(container, obj, obj,
headers={'x-object-meta-stats': obj})
retries_done += conn.attempts - 1
if report:
report(True)

39
bin/swift-stats-report
View File

@ -57,7 +57,8 @@ def get_error_log(prefix):
def audit(coropool, connpool, account, container_ring, object_ring, options):
begun = time()
with connpool.item() as conn:
estimated_items = [conn.head_account()[0]]
estimated_items = \
[int(conn.head_account()['x-account-container-count'])]
items_completed = [0]
retries_done = [0]
containers_missing_replicas = {}
@ -130,7 +131,8 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
cmarker = ''
while True:
with connpool.item() as conn:
containers = [c['name'] for c in conn.get_account(marker=cmarker)]
containers = \
[c['name'] for c in conn.get_account(marker=cmarker)[1]]
if not containers:
break
cmarker = containers[-1]
@ -142,7 +144,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
while True:
with connpool.item() as conn:
objects = [o['name'] for o in
conn.get_container(container, marker=omarker)]
conn.get_container(container, marker=omarker)[1]]
if not objects:
break
omarker = objects[-1]
@ -183,7 +185,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_dispersion_',
full_listing=True)]
full_listing=True)[1]]
containers_listed = len(containers)
if not containers_listed:
print >>stderr, 'No containers to query. Has stats-populate been run?'
@ -262,7 +264,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring, options):
with connpool.item() as conn:
try:
objects = [o['name'] for o in conn.get_container(container,
prefix='stats_object_dispersion_', full_listing=True)]
prefix='stats_object_dispersion_', full_listing=True)[1]]
except ClientException, err:
if err.http_status != 404:
raise
@ -384,7 +386,7 @@ def container_head_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def head(container):
with connpool.item() as conn:
@ -425,7 +427,7 @@ def container_get_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def get(container):
with connpool.item() as conn:
@ -463,7 +465,8 @@ def container_standard_listing_report(coropool, connpool, options):
print 'Listing big_container',
with connpool.item() as conn:
try:
value = len(conn.get_container('big_container', full_listing=True))
value = \
len(conn.get_container('big_container', full_listing=True)[1])
except ClientException, err:
if err.http_status != 404:
raise
@ -486,7 +489,7 @@ def container_prefix_listing_report(coropool, connpool, options):
try:
for x in xrange(256):
value += len(conn.get_container('big_container',
prefix=('%02x' % x), full_listing=True))
prefix=('%02x' % x), full_listing=True)[1])
except ClientException, err:
if err.http_status != 404:
raise
@ -511,7 +514,7 @@ def container_prefix_delimiter_listing_report(coropool, connpool, options):
try:
with connpool.item() as conn:
listing = conn.get_container('big_container',
marker=marker, prefix=prefix, delimiter='/')
marker=marker, prefix=prefix, delimiter='/')[1]
except ClientException, err:
if err.http_status != 404:
raise
@ -552,7 +555,7 @@ def container_delete_report(coropool, connpool, options):
with connpool.item() as conn:
containers = [c['name'] for c in
conn.get_account(prefix='stats_container_put_',
full_listing=True)]
full_listing=True)[1]]
count = len(containers)
def delete(container):
with connpool.item() as conn:
@ -630,7 +633,7 @@ def object_head_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def head(obj):
with connpool.item() as conn:
@ -670,7 +673,7 @@ def object_get_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def get(obj):
with connpool.item() as conn:
@ -710,7 +713,7 @@ def object_delete_report(coropool, connpool, options):
next_report = [time() + 2]
with connpool.item() as conn:
objects = [o['name'] for o in conn.get_container('stats_object_put',
prefix='stats_object_put_', full_listing=True)]
prefix='stats_object_put_', full_listing=True)[1]]
count = len(objects)
def delete(obj):
with connpool.item() as conn:
@ -791,7 +794,7 @@ Usage: %prog [options] [conf_file]
else:
options.retries = int(conf.get('retries', 5))
if not options.csv_output:
csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
coropool = GreenPool(size=concurrency)
@ -908,8 +911,8 @@ Usage: %prog [options] [conf_file]
if options.csv_output != 'None':
try:
if not os.path.exists(csv_output):
f = open(csv_output, 'wb')
if not os.path.exists(options.csv_output):
f = open(options.csv_output, 'wb')
f.write('Timestamp,'
'Container Dispersion Report Time,'
'Container Dispersion Report Value,'
@ -936,7 +939,7 @@ Usage: %prog [options] [conf_file]
'Object DELETE Report Success Rate\r\n')
csv = csv.writer(f)
else:
csv = csv.writer(open(csv_output, 'ab'))
csv = csv.writer(open(options.csv_output, 'ab'))
csv.writerow(report)
except Exception, err:
print >>stderr, 'Could not write CSV report:', err

203
swift/common/client.py
View File

@ -194,18 +194,21 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of accounts
:returns: a tuple of (response headers, a list of containers) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_account(url, token, marker, limit, prefix, http_conn)
rv = get_account(url, token, marker, limit, prefix, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
marker = listing[-1]['name']
listing = get_account(url, token, marker, limit, prefix, http_conn)
listing = \
get_account(url, token, marker, limit, prefix, http_conn)[1]
if listing:
rv.extend(listing)
return rv
parsed, conn = http_conn
qs = 'format=json'
@ -218,6 +221,9 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
conn.request('GET', '%s?%s' % (parsed.path, qs), '',
{'X-Auth-Token': token})
resp = conn.getresponse()
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status < 200 or resp.status >= 300:
resp.read()
raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@ -226,8 +232,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
http_reason=resp.reason)
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_account(url, token, http_conn=None):
@ -238,7 +244,8 @@ def head_account(url, token, http_conn=None):
:param token: auth token
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (container count, object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -253,9 +260,36 @@ def head_account(url, token, http_conn=None):
http_host=conn.host, http_port=conn.port,
http_path=parsed.path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-account-container-count', 0)), \
int(resp.getheader('x-account-object-count', 0)), \
int(resp.getheader('x-account-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def post_account(url, token, headers, http_conn=None):
"""
Update an account's metadata.
:param url: storage URL
:param token: auth token
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
headers['X-Auth-Token'] = token
conn.request('POST', parsed.path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Account POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def get_container(url, token, container, marker=None, limit=None,
@ -275,23 +309,25 @@ def get_container(url, token, container, marker=None, limit=None,
conn object)
:param full_listing: if True, return a full listing, else returns a max
of 10000 listings
:returns: a list of objects
:returns: a tuple of (response headers, a list of objects) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if not http_conn:
http_conn = http_connection(url)
if full_listing:
rv = []
listing = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
rv = get_container(url, token, container, marker, limit, prefix,
delimiter, http_conn)
listing = rv[1]
while listing:
rv.extend(listing)
if not delimiter:
marker = listing[-1]['name']
else:
marker = listing[-1].get('name', listing[-1].get('subdir'))
listing = get_container(url, token, container, marker, limit,
prefix, delimiter, http_conn)
prefix, delimiter, http_conn)[1]
if listing:
rv[1].extend(listing)
return rv
parsed, conn = http_conn
path = '%s/%s' % (parsed.path, quote(container))
@ -312,10 +348,13 @@ def get_container(url, token, container, marker=None, limit=None,
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_query=qs,
http_status=resp.status, http_reason=resp.reason)
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
if resp.status == 204:
resp.read()
return []
return json_loads(resp.read())
return resp_headers, []
return resp_headers, json_loads(resp.read())
def head_container(url, token, container, http_conn=None):
@ -327,7 +366,8 @@ def head_container(url, token, container, http_conn=None):
:param container: container name to get stats for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (object count, bytes used)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -343,17 +383,20 @@ def head_container(url, token, container, http_conn=None):
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
return int(resp.getheader('x-container-object-count', 0)), \
int(resp.getheader('x-container-bytes-used', 0))
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_container(url, token, container, http_conn=None):
def put_container(url, token, container, headers=None, http_conn=None):
"""
Create a container
:param url: storage URL
:param token: auth token
:param container: container name to create
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP PUT request failed
@ -363,7 +406,10 @@ def put_container(url, token, container, http_conn=None):
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
conn.request('PUT', path, '', {'X-Auth-Token': token})
if not headers:
headers = {}
headers['X-Auth-Token'] = token
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
@ -373,6 +419,34 @@ def put_container(url, token, container, http_conn=None):
http_reason=resp.reason)
def post_container(url, token, container, headers, http_conn=None):
"""
Update a container's metadata.
:param url: storage URL
:param token: auth token
:param container: container name to update
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
"""
if http_conn:
parsed, conn = http_conn
else:
parsed, conn = http_connection(url)
path = '%s/%s' % (parsed.path, quote(container))
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
if resp.status < 200 or resp.status >= 300:
raise ClientException('Container POST failed',
http_scheme=parsed.scheme, http_host=conn.host,
http_port=conn.port, http_path=path, http_status=resp.status,
http_reason=resp.reason)
def delete_container(url, token, container, http_conn=None):
"""
Delete a container
@ -410,8 +484,12 @@ def get_object(url, token, container, name, http_conn=None,
:param name: object name to get
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:param resp_chunk_size: if defined, chunk size of data to read
:returns: a list of objects
:param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
you specify a resp_chunk_size you must fully read
the object's contents before making another
request.
:returns: a tuple of (response headers, the object's contents) The response
headers will be a dict and all header names will be lowercase.
:raises ClientException: HTTP GET request failed
"""
if http_conn:
@ -440,12 +518,10 @@ def get_object(url, token, container, name, http_conn=None,
object_body = _object_body()
else:
object_body = resp.read()
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata, \
object_body
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers, object_body
def head_object(url, token, container, name, http_conn=None):
@ -458,8 +534,8 @@ def head_object(url, token, container, name, http_conn=None):
:param name: object name to get info for
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: a tuple of (content type, content length, last modfied, etag,
dictionary of metadata)
:returns: a dict containing the response's headers (all header names will
be lowercase)
:raises ClientException: HTTP HEAD request failed
"""
if http_conn:
@ -478,16 +554,15 @@ def head_object(url, token, container, name, http_conn=None):
for key, value in resp.getheaders():
if key.lower().startswith('x-object-meta-'):
metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
return resp.getheader('content-type'), \
int(resp.getheader('content-length', 0)), \
resp.getheader('last-modified'), \
resp.getheader('etag').strip('"'), \
metadata
resp_headers = {}
for header, value in resp.getheaders():
resp_headers[header.lower()] = value
return resp_headers
def put_object(url, token, container, name, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None, http_conn=None):
def put_object(url, token, container, name, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None, headers=None,
http_conn=None):
"""
Put an object
@ -495,12 +570,12 @@ def put_object(url, token, container, name, contents, metadata={},
:param token: auth token
:param container: container name that the object is in
:param name: object name to put
:param contents: file like object to read object data from
:param metadata: dictionary of object metadata
:param contents: a string or a file like object to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param chunk_size: chunk size of data to write
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:returns: etag from server response
@ -511,9 +586,9 @@ def put_object(url, token, container, name, contents, metadata={},
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
if not headers:
headers = {}
headers['X-Auth-Token'] = token
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
@ -549,7 +624,7 @@ def put_object(url, token, container, name, contents, metadata={},
return resp.getheader('etag').strip('"')
def post_object(url, token, container, name, metadata, http_conn=None):
def post_object(url, token, container, name, headers, http_conn=None):
"""
Change object metadata
@ -557,7 +632,7 @@ def post_object(url, token, container, name, metadata, http_conn=None):
:param token: auth token
:param container: container name that the object is in
:param name: object name to change
:param metadata: dictionary of object metadata
:param headers: additional headers to include in the request
:param http_conn: HTTP connection object (If None, it will create the
conn object)
:raises ClientException: HTTP POST request failed
@ -567,9 +642,7 @@ def post_object(url, token, container, name, metadata, http_conn=None):
else:
parsed, conn = http_connection(url)
path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
headers = {'X-Auth-Token': token}
for key, value in metadata.iteritems():
headers['X-Object-Meta-%s' % quote(key)] = quote(value)
headers['X-Auth-Token'] = token
conn.request('POST', path, '', headers)
resp = conn.getresponse()
resp.read()
@ -682,6 +755,10 @@ class Connection(object):
return self._retry(get_account, marker=marker, limit=limit,
prefix=prefix, full_listing=full_listing)
def post_account(self, headers):
"""Wrapper for :func:`post_account`"""
return self._retry(post_account, headers)
def head_container(self, container):
"""Wrapper for :func:`head_container`"""
return self._retry(head_container, container)
@ -696,9 +773,13 @@ class Connection(object):
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
def put_container(self, container):
def put_container(self, container, headers=None):
"""Wrapper for :func:`put_container`"""
return self._retry(put_container, container)
return self._retry(put_container, container, headers=headers)
def post_container(self, container, headers):
"""Wrapper for :func:`post_container`"""
return self._retry(post_container, container, headers)
def delete_container(self, container):
"""Wrapper for :func:`delete_container`"""
@ -713,17 +794,17 @@ class Connection(object):
return self._retry(get_object, container, obj,
resp_chunk_size=resp_chunk_size)
def put_object(self, container, obj, contents, metadata={},
content_length=None, etag=None, chunk_size=65536,
content_type=None):
def put_object(self, container, obj, contents, content_length=None,
etag=None, chunk_size=65536, content_type=None,
headers=None):
"""Wrapper for :func:`put_object`"""
return self._retry(put_object, container, obj, contents,
metadata=metadata, content_length=content_length, etag=etag,
chunk_size=chunk_size, content_type=content_type)
content_length=content_length, etag=etag, chunk_size=chunk_size,
content_type=content_type, headers=headers)
def post_object(self, container, obj, metadata):
def post_object(self, container, obj, headers):
"""Wrapper for :func:`post_object`"""
return self._retry(post_object, container, obj, metadata)
return self._retry(post_object, container, obj, headers)
def delete_object(self, container, obj):
"""Wrapper for :func:`delete_object`"""

46
test/probe/test_account_failures.py
View File

@ -39,8 +39,10 @@ class TestAccountFailures(unittest.TestCase):
client.put_container(self.url, self.token, container1)
container2 = 'container2'
client.put_container(self.url, self.token, container2)
self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for c in containers:
@ -56,8 +58,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
client.put_object(self.url, self.token, container2, 'object1', '1234')
self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '0')
self.assertEquals(headers['x-account-bytes-used'], '0')
found1 = False
found2 = False
for c in containers:
@ -73,8 +77,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
get_to_final_state()
containers = client.get_account(self.url, self.token)
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -94,8 +100,10 @@ class TestAccountFailures(unittest.TestCase):
client.delete_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container2, 'object2', '12345')
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -115,8 +123,10 @@ class TestAccountFailures(unittest.TestCase):
'once']))
for p in ps:
p.wait()
self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
containers = client.get_account(self.url, self.token)
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for c in containers:
@ -134,10 +144,12 @@ class TestAccountFailures(unittest.TestCase):
'/etc/swift/account-server/%d.conf' %
((anodes[0]['port'] - 6002) / 10)]).pid
sleep(2)
# This is the earlier object count and bytes because the first node
# doesn't have the newest udpates yet.
self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
containers = client.get_account(self.url, self.token)
# This is the earlier counts and bytes because the first node doesn't
# have the newest udpates yet.
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '2')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
found2 = False
for c in containers:
@ -155,8 +167,10 @@ class TestAccountFailures(unittest.TestCase):
self.assert_(found2)
get_to_final_state()
containers = client.get_account(self.url, self.token)
self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
headers, containers = client.get_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '2')
self.assertEquals(headers['x-account-bytes-used'], '9')
found1 = False
found2 = False
for c in containers:

104
test/probe/test_container_failures.py
View File

@ -40,23 +40,23 @@ class TestContainerFailures(unittest.TestCase):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[0]['port']]] = \
Popen(['swift-container-server',
@ -64,11 +64,11 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[0]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -87,7 +87,7 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
@ -95,44 +95,44 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_second_node_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.pids[self.port2server[cnodes[1]['port']]] = \
Popen(['swift-container-server',
@ -140,9 +140,9 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[1]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -161,42 +161,42 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
@ -204,9 +204,9 @@ class TestContainerFailures(unittest.TestCase):
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
@ -215,11 +215,11 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -238,7 +238,7 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
@ -246,35 +246,35 @@ class TestContainerFailures(unittest.TestCase):
client.put_object(self.url, self.token, container, object2, 'test')
# First node still doesn't know object1 was deleted yet; this is okay.
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
def test_last_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in (1, 2):
@ -282,9 +282,9 @@ class TestContainerFailures(unittest.TestCase):
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
for x in (1, 2):
self.pids[self.port2server[cnodes[x]['port']]] = \
@ -293,9 +293,9 @@ class TestContainerFailures(unittest.TestCase):
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
@ -314,29 +314,29 @@ class TestContainerFailures(unittest.TestCase):
# account server, this'll pass, otherwise the first account server will
# serve the listing and not have the container.
# self.assert_(container in [c['name'] for c in
# client.get_account(self.url, self.token)])
# client.get_account(self.url, self.token)[1]])
object2 = 'object2'
# This will work because at least one (in this case, just one) account
# server has to indicate the container exists for the put to continue.
client.put_object(self.url, self.token, container, object2, 'test')
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
# And, of course, our new object2 exists.
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
get_to_final_state()
# Our container delete never "finalized" because we started using it
# before the delete settled.
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)])
client.get_account(self.url, self.token)[1]])
# And, so our object2 should still exist and object1's delete should
# have finalized.
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
self.assert_(object2 in [o['name'] for o in
client.get_container(self.url, self.token, container)])
client.get_container(self.url, self.token, container)[1]])
if __name__ == '__main__':

11
test/probe/test_object_handoff.py
View File

@ -75,7 +75,7 @@ class TestObjectHandoff(unittest.TestCase):
raise Exception('Direct object GET did not return VERIFY, instead '
'it returned: %s' % repr(odata))
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)]
client.get_container(self.url, self.token, container)[1]]
if obj not in objs:
raise Exception('Container listing did not know about object')
for cnode in cnodes:
@ -126,10 +126,9 @@ class TestObjectHandoff(unittest.TestCase):
kill(self.pids[self.port2server[onode['port']]], SIGTERM)
client.post_object(self.url, self.token, container, obj,
{'probe': 'value'})
ometadata = client.head_object(
self.url, self.token, container, obj)[-1]
if ometadata.get('probe') != 'value':
headers={'x-object-meta-probe': 'value'})
ometadata = client.head_object(self.url, self.token, container, obj)
if ometadata.get('x-object-meta-probe') != 'value':
raise Exception('Metadata incorrect, was %s' % repr(ometadata))
exc = False
try:
@ -177,7 +176,7 @@ class TestObjectHandoff(unittest.TestCase):
if not exc:
raise Exception('Regular object HEAD was still successful')
objs = [o['name'] for o in
client.get_container(self.url, self.token, container)]
client.get_container(self.url, self.token, container)[1]]
if obj in objs:
raise Exception('Container listing still knew about object')
for cnode in cnodes:

27
test/probe/test_running_with_each_type_down.py
View File

@ -56,16 +56,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
pass
client.put_object(self.url, self.token, 'container1', 'object1', '1234')
get_to_final_state()
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
@ -84,15 +87,18 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
'/etc/swift/object-server/%d.conf' %
((onodes[0]['port'] - 6000) / 10)]).pid
sleep(2)
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
# The account node was previously down.
self.assert_(not found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)
@ -101,16 +107,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
self.assert_(found1)
get_to_final_state()
self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
headers, containers = client.head_account(self.url, self.token)
self.assertEquals(headers['x-account-container-count'], '1')
self.assertEquals(headers['x-account-object-count'], '1')
self.assertEquals(headers['x-account-bytes-used'], '4')
found1 = False
for container in client.get_account(self.url, self.token):
for container in containers:
if container['name'] == 'container1':
found1 = True
self.assertEquals(container['count'], 1)
self.assertEquals(container['bytes'], 4)
self.assert_(found1)
found1 = False
for obj in client.get_container(self.url, self.token, 'container1'):
for obj in client.get_container(self.url, self.token, 'container1')[1]:
if obj['name'] == 'object1':
found1 = True
self.assertEquals(obj['bytes'], 4)

9
test/unit/common/test_client.py
View File

@ -191,7 +191,7 @@ class TestGetAccount(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_account('http://www.test.com', 'asdf')
value = c.get_account('http://www.test.com', 'asdf')[1]
self.assertEquals(value, [])
@ -200,7 +200,10 @@ class TestHeadAccount(MockHttpTest):
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.head_account('http://www.tests.com', 'asdf')
self.assertEquals(value, (0, 0, 0))
# TODO: Hmm. This doesn't really test too much as it uses a fake that
# always returns the same dict. I guess it "exercises" the code, so
# I'll leave it for now.
self.assertEquals(type(value), dict)
def test_server_error(self):
c.http_connection = self.fake_http_connection(500)
@ -212,7 +215,7 @@ class TestGetContainer(MockHttpTest):
def test_no_content(self):
c.http_connection = self.fake_http_connection(204)
value = c.get_container('http://www.test.com', 'asdf', 'asdf')
value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1]
self.assertEquals(value, [])

4
test/unit/proxy/test_server.py
View File

@ -82,7 +82,7 @@ def fake_http_connect(*code_iter, **kwargs):
pass
if 'slow' in kwargs:
headers['content-length'] = '4'
return headers
return headers.items()
def read(self, amt=None):
if 'slow' in kwargs:
if self.sent < 4:
@ -98,7 +98,7 @@ def fake_http_connect(*code_iter, **kwargs):
self.received += 1
sleep(0.1)
def getheader(self, name, default=None):
return self.getheaders().get(name.lower(), default)
return dict(self.getheaders()).get(name.lower(), default)
etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
x = kwargs.get('missing_container', [False] * len(code_iter))
if not isinstance(x, (tuple, list)):