diff --git a/bin/st b/bin/st index e65a0ac0b9..6a8b02bb37 100755 --- a/bin/st +++ b/bin/st @@ -14,807 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - # Try to use installed swift.common.client... - from swift.common.client import get_auth, ClientException, Connection -except: - # But if not installed, use an included copy. - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - # Inclusion of swift.common.client - - """ - Cloud Files client library used internally - """ - import socket - from cStringIO import StringIO - from httplib import HTTPConnection, HTTPException, HTTPSConnection - from re import compile, DOTALL - from tokenize import generate_tokens, STRING, NAME, OP - from urllib import quote as _quote, unquote - from urlparse import urlparse, urlunparse - - try: - from eventlet import sleep - except: - from time import sleep - - - def quote(value, safe='/'): - """ - Patched version of urllib.quote that encodes utf8 strings before quoting - """ - if isinstance(value, unicode): - value = value.encode('utf8') - return _quote(value, safe) - - - # look for a real json parser first - try: - # simplejson is popular and pretty good - from simplejson import loads as json_loads - except ImportError: - try: - # 2.6 will have a json module in the stdlib - from json import loads as json_loads - except ImportError: - # fall back on local parser otherwise - comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) - - def json_loads(string): - ''' - Fairly competent json parser exploiting the python tokenizer and - eval(). -- From python-cloudfiles - - _loads(serialized_json) -> object - ''' - try: - res = [] - consts = {'true': True, 'false': False, 'null': None} - string = '(' + comments.sub('', string) + ')' - for type, val, _, _, _ in \ - generate_tokens(StringIO(string).readline): - if (type == OP and val not in '[]{}:,()-') or \ - (type == NAME and val not in consts): - raise AttributeError() - elif type == STRING: - res.append('u') - res.append(val.replace('\\/', '/')) - else: - res.append(val) - return eval(''.join(res), {}, consts) - except: - raise AttributeError() - - - class ClientException(Exception): - - def __init__(self, msg, http_scheme='', http_host='', http_port='', - http_path='', http_query='', http_status=0, http_reason='', - http_device=''): - Exception.__init__(self, msg) - self.msg = msg - self.http_scheme = http_scheme - self.http_host = http_host - self.http_port = http_port - self.http_path = http_path - self.http_query = http_query - self.http_status = http_status - self.http_reason = http_reason - self.http_device = http_device - - def __str__(self): - a = self.msg - b = '' - if self.http_scheme: - b += '%s://' % self.http_scheme - if self.http_host: - b += self.http_host - if self.http_port: - b += ':%s' % self.http_port - if self.http_path: - b += self.http_path - if self.http_query: - b += '?%s' % self.http_query - if self.http_status: - if b: - b = '%s %s' % (b, self.http_status) - else: - b = str(self.http_status) - if self.http_reason: - if b: - b = '%s %s' % (b, self.http_reason) - else: - b = '- %s' % self.http_reason - if self.http_device: - if b: - b = '%s: device %s' % (b, self.http_device) - else: - b = 'device %s' % self.http_device - return b and '%s: %s' % (a, b) or a - - - def http_connection(url): - """ - Make an HTTPConnection or HTTPSConnection - - :param url: url to connect to - :returns: tuple of (parsed url, connection object) - :raises ClientException: Unable to handle protocol scheme - """ - parsed = urlparse(url) - if parsed.scheme == 'http': - conn = HTTPConnection(parsed.netloc) - elif parsed.scheme == 'https': - conn = HTTPSConnection(parsed.netloc) - else: - raise ClientException('Cannot handle protocol scheme %s for url %s' % - (parsed.scheme, repr(url))) - return parsed, conn - - - def get_auth(url, user, key, snet=False): - """ - Get authentication/authorization credentials. - - The snet parameter is used for Rackspace's ServiceNet internal network - implementation. In this function, it simply adds *snet-* to the beginning - of the host name for the returned storage URL. With Rackspace Cloud Files, - use of this network path causes no bandwidth charges but requires the - client to be running on Rackspace's ServiceNet network. - - :param url: authentication/authorization URL - :param user: user to authenticate as - :param key: key or password for authorization - :param snet: use SERVICENET internal network (see above), default is False - :returns: tuple of (storage URL, auth token) - :raises ClientException: HTTP GET request to auth URL failed - """ - parsed, conn = http_connection(url) - conn.request('GET', parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_status=resp.status, - http_reason=resp.reason) - url = resp.getheader('x-storage-url') - if snet: - parsed = list(urlparse(url)) - # Second item in the list is the netloc - parsed[1] = 'snet-' + parsed[1] - url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) - - - def get_account(url, token, marker=None, limit=None, prefix=None, - http_conn=None, full_listing=False): - """ - Get a listing of containers for the account. - - :param url: storage URL - :param token: auth token - :param marker: marker query - :param limit: limit query - :param prefix: prefix query - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param full_listing: if True, return a full listing, else returns a max - of 10000 listings - :returns: a tuple of (response headers, a list of containers) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if not http_conn: - http_conn = http_connection(url) - if full_listing: - rv = get_account(url, token, marker, limit, prefix, http_conn) - listing = rv[1] - while listing: - marker = listing[-1]['name'] - listing = \ - get_account(url, token, marker, limit, prefix, http_conn)[1] - if listing: - rv.extend(listing) - return rv - parsed, conn = http_conn - qs = 'format=json' - if marker: - qs += '&marker=%s' % quote(marker) - if limit: - qs += '&limit=%d' % limit - if prefix: - qs += '&prefix=%s' % quote(prefix) - conn.request('GET', '%s?%s' % (parsed.path, qs), '', - {'X-Auth-Token': token}) - resp = conn.getresponse() - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - if resp.status < 200 or resp.status >= 300: - resp.read() - raise ClientException('Account GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_query=qs, http_status=resp.status, - http_reason=resp.reason) - if resp.status == 204: - resp.read() - return resp_headers, [] - return resp_headers, json_loads(resp.read()) - - - def head_account(url, token, http_conn=None): - """ - Get account stats. - - :param url: storage URL - :param token: auth token - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, - http_path=parsed.path, http_status=resp.status, - http_reason=resp.reason) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - - def post_account(url, token, headers, http_conn=None): - """ - Update an account's metadata. - - :param url: storage URL - :param token: auth token - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - headers['X-Auth-Token'] = token - conn.request('POST', parsed.path, '', headers) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Account POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - - - def get_container(url, token, container, marker=None, limit=None, - prefix=None, delimiter=None, http_conn=None, - full_listing=False): - """ - Get a listing of objects for the container. - - :param url: storage URL - :param token: auth token - :param container: container name to get a listing for - :param marker: marker query - :param limit: limit query - :param prefix: prefix query - :param delimeter: string to delimit the queries on - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param full_listing: if True, return a full listing, else returns a max - of 10000 listings - :returns: a tuple of (response headers, a list of objects) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if not http_conn: - http_conn = http_connection(url) - if full_listing: - rv = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) - listing = rv[1] - while listing: - if not delimiter: - marker = listing[-1]['name'] - else: - marker = listing[-1].get('name', listing[-1].get('subdir')) - listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn)[1] - if listing: - rv[1].extend(listing) - return rv - parsed, conn = http_conn - path = '%s/%s' % (parsed.path, quote(container)) - qs = 'format=json' - if marker: - qs += '&marker=%s' % quote(marker) - if limit: - qs += '&limit=%d' % limit - if prefix: - qs += '&prefix=%s' % quote(prefix) - if delimiter: - qs += '&delimiter=%s' % quote(delimiter) - conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) - resp = conn.getresponse() - if resp.status < 200 or resp.status >= 300: - resp.read() - raise ClientException('Container GET failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_query=qs, - http_status=resp.status, http_reason=resp.reason) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - if resp.status == 204: - resp.read() - return resp_headers, [] - return resp_headers, json_loads(resp.read()) - - - def head_container(url, token, container, http_conn=None): - """ - Get container stats. - - :param url: storage URL - :param token: auth token - :param container: container name to get stats for - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - conn.request('HEAD', path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container HEAD failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - - def put_container(url, token, container, headers=None, http_conn=None): - """ - Create a container - - :param url: storage URL - :param token: auth token - :param container: container name to create - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP PUT request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - if not headers: - headers = {} - headers['X-Auth-Token'] = token - conn.request('PUT', path, '', headers) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container PUT failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - - - def post_container(url, token, container, headers, http_conn=None): - """ - Update a container's metadata. - - :param url: storage URL - :param token: auth token - :param container: container name to update - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - headers['X-Auth-Token'] = token - conn.request('POST', path, '', headers) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - - - def delete_container(url, token, container, http_conn=None): - """ - Delete a container - - :param url: storage URL - :param token: auth token - :param container: container name to delete - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP DELETE request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s' % (parsed.path, quote(container)) - conn.request('DELETE', path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Container DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - - - def get_object(url, token, container, name, http_conn=None, - resp_chunk_size=None): - """ - Get an object - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to get - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If - you specify a resp_chunk_size you must fully read - the object's contents before making another - request. - :returns: a tuple of (response headers, the object's contents) The response - headers will be a dict and all header names will be lowercase. - :raises ClientException: HTTP GET request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - conn.request('GET', path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - if resp.status < 200 or resp.status >= 300: - resp.read() - raise ClientException('Object GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason) - if resp_chunk_size: - - def _object_body(): - buf = resp.read(resp_chunk_size) - while buf: - yield buf - buf = resp.read(resp_chunk_size) - object_body = _object_body() - else: - object_body = resp.read() - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers, object_body - - - def head_object(url, token, container, name, http_conn=None): - """ - Get object info - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to get info for - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: a dict containing the response's headers (all header names will - be lowercase) - :raises ClientException: HTTP HEAD request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - conn.request('HEAD', path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - return resp_headers - - - def put_object(url, token, container, name, contents, content_length=None, - etag=None, chunk_size=65536, content_type=None, headers=None, - http_conn=None): - """ - Put an object - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to put - :param contents: a string or a file like object to read object data from - :param content_length: value to send as content-length header - :param etag: etag of contents - :param chunk_size: chunk size of data to write - :param content_type: value to send as content-type header - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :returns: etag from server response - :raises ClientException: HTTP PUT request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - if not headers: - headers = {} - headers['X-Auth-Token'] = token - if etag: - headers['ETag'] = etag.strip('"') - if content_length is not None: - headers['Content-Length'] = str(content_length) - if content_type is not None: - headers['Content-Type'] = content_type - if not contents: - headers['Content-Length'] = '0' - if hasattr(contents, 'read'): - conn.putrequest('PUT', path) - for header, value in headers.iteritems(): - conn.putheader(header, value) - if not content_length: - conn.putheader('Transfer-Encoding', 'chunked') - conn.endheaders() - chunk = contents.read(chunk_size) - while chunk: - if not content_length: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - else: - conn.send(chunk) - chunk = contents.read(chunk_size) - if not content_length: - conn.send('0\r\n\r\n') - else: - conn.request('PUT', path, contents, headers) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object PUT failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason) - return resp.getheader('etag').strip('"') - - - def post_object(url, token, container, name, headers, http_conn=None): - """ - Update object metadata - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: name of the object to update - :param headers: additional headers to include in the request - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP POST request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token - conn.request('POST', path, '', headers) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object POST failed', http_scheme=parsed.scheme, - http_host=conn.host, http_port=conn.port, http_path=path, - http_status=resp.status, http_reason=resp.reason) - - - def delete_object(url, token, container, name, http_conn=None): - """ - Delete object - - :param url: storage URL - :param token: auth token - :param container: container name that the object is in - :param name: object name to delete - :param http_conn: HTTP connection object (If None, it will create the - conn object) - :raises ClientException: HTTP DELETE request failed - """ - if http_conn: - parsed, conn = http_conn - else: - parsed, conn = http_connection(url) - path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - conn.request('DELETE', path, '', {'X-Auth-Token': token}) - resp = conn.getresponse() - resp.read() - if resp.status < 200 or resp.status >= 300: - raise ClientException('Object DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_port=conn.port, http_path=path, http_status=resp.status, - http_reason=resp.reason) - - - class Connection(object): - """Convenience class to make requests that will also retry the request""" - - def __init__(self, authurl, user, key, retries=5, preauthurl=None, - preauthtoken=None, snet=False): - """ - :param authurl: authenitcation URL - :param user: user name to authenticate as - :param key: key/password to authenticate with - :param retries: Number of times to retry the request before failing - :param preauthurl: storage URL (if you have already authenticated) - :param preauthtoken: authentication token (if you have already - authenticated) - :param snet: use SERVICENET internal network default is False - """ - self.authurl = authurl - self.user = user - self.key = key - self.retries = retries - self.http_conn = None - self.url = preauthurl - self.token = preauthtoken - self.attempts = 0 - self.snet = snet - - def get_auth(self): - return get_auth(self.authurl, self.user, self.key, snet=self.snet) - - def http_connection(self): - return http_connection(self.url) - - def _retry(self, func, *args, **kwargs): - self.attempts = 0 - backoff = 1 - while self.attempts <= self.retries: - self.attempts += 1 - try: - if not self.url or not self.token: - self.url, self.token = self.get_auth() - self.http_conn = None - if not self.http_conn: - self.http_conn = self.http_connection() - kwargs['http_conn'] = self.http_conn - rv = func(self.url, self.token, *args, **kwargs) - return rv - except (socket.error, HTTPException): - if self.attempts > self.retries: - raise - self.http_conn = None - except ClientException, err: - if self.attempts > self.retries: - raise - if err.http_status == 401: - self.url = self.token = None - if self.attempts > 1: - raise - elif 500 <= err.http_status <= 599: - pass - else: - raise - sleep(backoff) - backoff *= 2 - - def head_account(self): - """Wrapper for :func:`head_account`""" - return self._retry(head_account) - - def get_account(self, marker=None, limit=None, prefix=None, - full_listing=False): - """Wrapper for :func:`get_account`""" - # TODO(unknown): With full_listing=True this will restart the entire - # listing with each retry. Need to make a better version that just - # retries where it left off. - return self._retry(get_account, marker=marker, limit=limit, - prefix=prefix, full_listing=full_listing) - - def post_account(self, headers): - """Wrapper for :func:`post_account`""" - return self._retry(post_account, headers) - - def head_container(self, container): - """Wrapper for :func:`head_container`""" - return self._retry(head_container, container) - - def get_container(self, container, marker=None, limit=None, prefix=None, - delimiter=None, full_listing=False): - """Wrapper for :func:`get_container`""" - # TODO(unknown): With full_listing=True this will restart the entire - # listing with each retry. Need to make a better version that just - # retries where it left off. - return self._retry(get_container, container, marker=marker, - limit=limit, prefix=prefix, delimiter=delimiter, - full_listing=full_listing) - - def put_container(self, container, headers=None): - """Wrapper for :func:`put_container`""" - return self._retry(put_container, container, headers=headers) - - def post_container(self, container, headers): - """Wrapper for :func:`post_container`""" - return self._retry(post_container, container, headers) - - def delete_container(self, container): - """Wrapper for :func:`delete_container`""" - return self._retry(delete_container, container) - - def head_object(self, container, obj): - """Wrapper for :func:`head_object`""" - return self._retry(head_object, container, obj) - - def get_object(self, container, obj, resp_chunk_size=None): - """Wrapper for :func:`get_object`""" - return self._retry(get_object, container, obj, - resp_chunk_size=resp_chunk_size) - - def put_object(self, container, obj, contents, content_length=None, - etag=None, chunk_size=65536, content_type=None, - headers=None): - """Wrapper for :func:`put_object`""" - return self._retry(put_object, container, obj, contents, - content_length=content_length, etag=etag, chunk_size=chunk_size, - content_type=content_type, headers=headers) - - def post_object(self, container, obj, headers): - """Wrapper for :func:`post_object`""" - return self._retry(post_object, container, obj, headers) - - def delete_object(self, container, obj): - """Wrapper for :func:`delete_object`""" - return self._retry(delete_object, container, obj) - - # End inclusion of swift.common.client - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - - from errno import EEXIST, ENOENT from hashlib import md5 from optparse import OptionParser @@ -826,6 +25,805 @@ from threading import enumerate as threading_enumerate, Thread from time import sleep +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Inclusion of swift.common.client for convenience of single file distribution + +import socket +from cStringIO import StringIO +from httplib import HTTPException, HTTPSConnection +from re import compile, DOTALL +from tokenize import generate_tokens, STRING, NAME, OP +from urllib import quote as _quote, unquote +from urlparse import urlparse, urlunparse + +try: + from eventlet import sleep +except: + from time import sleep + +try: + from swift.common.bufferedhttp \ + import BufferedHTTPConnection as HTTPConnection +except: + from httplib import HTTPConnection + + +def quote(value, safe='/'): + """ + Patched version of urllib.quote that encodes utf8 strings before quoting + """ + if isinstance(value, unicode): + value = value.encode('utf8') + return _quote(value, safe) + + +# look for a real json parser first +try: + # simplejson is popular and pretty good + from simplejson import loads as json_loads +except ImportError: + try: + # 2.6 will have a json module in the stdlib + from json import loads as json_loads + except ImportError: + # fall back on local parser otherwise + comments = compile(r'/\*.*\*/|//[^\r\n]*', DOTALL) + + def json_loads(string): + ''' + Fairly competent json parser exploiting the python tokenizer and + eval(). -- From python-cloudfiles + + _loads(serialized_json) -> object + ''' + try: + res = [] + consts = {'true': True, 'false': False, 'null': None} + string = '(' + comments.sub('', string) + ')' + for type, val, _, _, _ in \ + generate_tokens(StringIO(string).readline): + if (type == OP and val not in '[]{}:,()-') or \ + (type == NAME and val not in consts): + raise AttributeError() + elif type == STRING: + res.append('u') + res.append(val.replace('\\/', '/')) + else: + res.append(val) + return eval(''.join(res), {}, consts) + except: + raise AttributeError() + + +class ClientException(Exception): + + def __init__(self, msg, http_scheme='', http_host='', http_port='', + http_path='', http_query='', http_status=0, http_reason='', + http_device=''): + Exception.__init__(self, msg) + self.msg = msg + self.http_scheme = http_scheme + self.http_host = http_host + self.http_port = http_port + self.http_path = http_path + self.http_query = http_query + self.http_status = http_status + self.http_reason = http_reason + self.http_device = http_device + + def __str__(self): + a = self.msg + b = '' + if self.http_scheme: + b += '%s://' % self.http_scheme + if self.http_host: + b += self.http_host + if self.http_port: + b += ':%s' % self.http_port + if self.http_path: + b += self.http_path + if self.http_query: + b += '?%s' % self.http_query + if self.http_status: + if b: + b = '%s %s' % (b, self.http_status) + else: + b = str(self.http_status) + if self.http_reason: + if b: + b = '%s %s' % (b, self.http_reason) + else: + b = '- %s' % self.http_reason + if self.http_device: + if b: + b = '%s: device %s' % (b, self.http_device) + else: + b = 'device %s' % self.http_device + return b and '%s: %s' % (a, b) or a + + +def http_connection(url): + """ + Make an HTTPConnection or HTTPSConnection + + :param url: url to connect to + :returns: tuple of (parsed url, connection object) + :raises ClientException: Unable to handle protocol scheme + """ + parsed = urlparse(url) + if parsed.scheme == 'http': + conn = HTTPConnection(parsed.netloc) + elif parsed.scheme == 'https': + conn = HTTPSConnection(parsed.netloc) + else: + raise ClientException('Cannot handle protocol scheme %s for url %s' % + (parsed.scheme, repr(url))) + return parsed, conn + + +def get_auth(url, user, key, snet=False): + """ + Get authentication/authorization credentials. + + The snet parameter is used for Rackspace's ServiceNet internal network + implementation. In this function, it simply adds *snet-* to the beginning + of the host name for the returned storage URL. With Rackspace Cloud Files, + use of this network path causes no bandwidth charges but requires the + client to be running on Rackspace's ServiceNet network. + + :param url: authentication/authorization URL + :param user: user to authenticate as + :param key: key or password for authorization + :param snet: use SERVICENET internal network (see above), default is False + :returns: tuple of (storage URL, auth token) + :raises ClientException: HTTP GET request to auth URL failed + """ + parsed, conn = http_connection(url) + conn.request('GET', parsed.path, '', + {'X-Auth-User': user, 'X-Auth-Key': key}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Auth GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + url = resp.getheader('x-storage-url') + if snet: + parsed = list(urlparse(url)) + # Second item in the list is the netloc + parsed[1] = 'snet-' + parsed[1] + url = urlunparse(parsed) + return url, resp.getheader('x-storage-token', + resp.getheader('x-auth-token')) + + +def get_account(url, token, marker=None, limit=None, prefix=None, + http_conn=None, full_listing=False): + """ + Get a listing of containers for the account. + + :param url: storage URL + :param token: auth token + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = get_account(url, token, marker, limit, prefix, http_conn) + listing = rv[1] + while listing: + marker = listing[-1]['name'] + listing = \ + get_account(url, token, marker, limit, prefix, http_conn)[1] + if listing: + rv.extend(listing) + return rv + parsed, conn = http_conn + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + conn.request('GET', '%s?%s' % (parsed.path, qs), '', + {'X-Auth-Token': token}) + resp = conn.getresponse() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Account GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_query=qs, http_status=resp.status, + http_reason=resp.reason) + if resp.status == 204: + resp.read() + return resp_headers, [] + return resp_headers, json_loads(resp.read()) + + +def head_account(url, token, http_conn=None): + """ + Get account stats. + + :param url: storage URL + :param token: auth token + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a dict containing the response's headers (all header names will + be lowercase) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, + http_path=parsed.path, http_status=resp.status, + http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + +def post_account(url, token, headers, http_conn=None): + """ + Update an account's metadata. + + :param url: storage URL + :param token: auth token + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + headers['X-Auth-Token'] = token + conn.request('POST', parsed.path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def get_container(url, token, container, marker=None, limit=None, + prefix=None, delimiter=None, http_conn=None, + full_listing=False): + """ + Get a listing of objects for the container. + + :param url: storage URL + :param token: auth token + :param container: container name to get a listing for + :param marker: marker query + :param limit: limit query + :param prefix: prefix query + :param delimeter: string to delimit the queries on + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param full_listing: if True, return a full listing, else returns a max + of 10000 listings + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. + :raises ClientException: HTTP GET request failed + """ + if not http_conn: + http_conn = http_connection(url) + if full_listing: + rv = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + listing = rv[1] + while listing: + if not delimiter: + marker = listing[-1]['name'] + else: + marker = listing[-1].get('name', listing[-1].get('subdir')) + listing = get_container(url, token, container, marker, limit, + prefix, delimiter, http_conn)[1] + if listing: + rv[1].extend(listing) + return rv + parsed, conn = http_conn + path = '%s/%s' % (parsed.path, quote(container)) + qs = 'format=json' + if marker: + qs += '&marker=%s' % quote(marker) + if limit: + qs += '&limit=%d' % limit + if prefix: + qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) + conn.request('GET', '%s?%s' % (path, qs), '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Container GET failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_query=qs, + http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + if resp.status == 204: + resp.read() + return resp_headers, [] + return resp_headers, json_loads(resp.read()) + + +def head_container(url, token, container, http_conn=None): + """ + Get container stats. + + :param url: storage URL + :param token: auth token + :param container: container name to get stats for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a dict containing the response's headers (all header names will + be lowercase) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container HEAD failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + +def put_container(url, token, container, headers=None, http_conn=None): + """ + Create a container + + :param url: storage URL + :param token: auth token + :param container: container name to create + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + conn.request('PUT', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container PUT failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def post_container(url, token, container, headers, http_conn=None): + """ + Update a container's metadata. + + :param url: storage URL + :param token: auth token + :param container: container name to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def delete_container(url, token, container, http_conn=None): + """ + Delete a container + + :param url: storage URL + :param token: auth token + :param container: container name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +def get_object(url, token, container, name, http_conn=None, + resp_chunk_size=None): + """ + Get an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If + you specify a resp_chunk_size you must fully read + the object's contents before making another + request. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. + :raises ClientException: HTTP GET request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('GET', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + if resp.status < 200 or resp.status >= 300: + resp.read() + raise ClientException('Object GET failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body + + +def head_object(url, token, container, name, http_conn=None): + """ + Get object info + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to get info for + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: a dict containing the response's headers (all header names will + be lowercase) + :raises ClientException: HTTP HEAD request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('HEAD', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + +def put_object(url, token, container, name, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, headers=None, + http_conn=None): + """ + Put an object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to put + :param contents: a string or a file like object to read object data from + :param content_length: value to send as content-length header + :param etag: etag of contents + :param chunk_size: chunk size of data to write + :param content_type: value to send as content-type header + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :returns: etag from server response + :raises ClientException: HTTP PUT request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + if etag: + headers['ETag'] = etag.strip('"') + if content_length is not None: + headers['Content-Length'] = str(content_length) + if content_type is not None: + headers['Content-Type'] = content_type + if not contents: + headers['Content-Length'] = '0' + if hasattr(contents, 'read'): + conn.putrequest('PUT', path) + for header, value in headers.iteritems(): + conn.putheader(header, value) + if not content_length: + conn.putheader('Transfer-Encoding', 'chunked') + conn.endheaders() + chunk = contents.read(chunk_size) + while chunk: + if not content_length: + conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) + else: + conn.send(chunk) + chunk = contents.read(chunk_size) + if not content_length: + conn.send('0\r\n\r\n') + else: + conn.request('PUT', path, contents, headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object PUT failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + return resp.getheader('etag').strip('"') + + +def post_object(url, token, container, name, headers, http_conn=None): + """ + Update object metadata + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: name of the object to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object POST failed', http_scheme=parsed.scheme, + http_host=conn.host, http_port=conn.port, http_path=path, + http_status=resp.status, http_reason=resp.reason) + + +def delete_object(url, token, container, name, http_conn=None): + """ + Delete object + + :param url: storage URL + :param token: auth token + :param container: container name that the object is in + :param name: object name to delete + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP DELETE request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + conn.request('DELETE', path, '', {'X-Auth-Token': token}) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Object DELETE failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + +class Connection(object): + """Convenience class to make requests that will also retry the request""" + + def __init__(self, authurl, user, key, retries=5, preauthurl=None, + preauthtoken=None, snet=False): + """ + :param authurl: authenitcation URL + :param user: user name to authenticate as + :param key: key/password to authenticate with + :param retries: Number of times to retry the request before failing + :param preauthurl: storage URL (if you have already authenticated) + :param preauthtoken: authentication token (if you have already + authenticated) + :param snet: use SERVICENET internal network default is False + """ + self.authurl = authurl + self.user = user + self.key = key + self.retries = retries + self.http_conn = None + self.url = preauthurl + self.token = preauthtoken + self.attempts = 0 + self.snet = snet + + def get_auth(self): + return get_auth(self.authurl, self.user, self.key, snet=self.snet) + + def http_connection(self): + return http_connection(self.url) + + def _retry(self, func, *args, **kwargs): + self.attempts = 0 + backoff = 1 + while self.attempts <= self.retries: + self.attempts += 1 + try: + if not self.url or not self.token: + self.url, self.token = self.get_auth() + self.http_conn = None + if not self.http_conn: + self.http_conn = self.http_connection() + kwargs['http_conn'] = self.http_conn + rv = func(self.url, self.token, *args, **kwargs) + return rv + except (socket.error, HTTPException): + if self.attempts > self.retries: + raise + self.http_conn = None + except ClientException, err: + if self.attempts > self.retries: + raise + if err.http_status == 401: + self.url = self.token = None + if self.attempts > 1: + raise + elif 500 <= err.http_status <= 599: + pass + else: + raise + sleep(backoff) + backoff *= 2 + + def head_account(self): + """Wrapper for :func:`head_account`""" + return self._retry(head_account) + + def get_account(self, marker=None, limit=None, prefix=None, + full_listing=False): + """Wrapper for :func:`get_account`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. + return self._retry(get_account, marker=marker, limit=limit, + prefix=prefix, full_listing=full_listing) + + def post_account(self, headers): + """Wrapper for :func:`post_account`""" + return self._retry(post_account, headers) + + def head_container(self, container): + """Wrapper for :func:`head_container`""" + return self._retry(head_container, container) + + def get_container(self, container, marker=None, limit=None, prefix=None, + delimiter=None, full_listing=False): + """Wrapper for :func:`get_container`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. + return self._retry(get_container, container, marker=marker, + limit=limit, prefix=prefix, delimiter=delimiter, + full_listing=full_listing) + + def put_container(self, container, headers=None): + """Wrapper for :func:`put_container`""" + return self._retry(put_container, container, headers=headers) + + def post_container(self, container, headers): + """Wrapper for :func:`post_container`""" + return self._retry(post_container, container, headers) + + def delete_container(self, container): + """Wrapper for :func:`delete_container`""" + return self._retry(delete_container, container) + + def head_object(self, container, obj): + """Wrapper for :func:`head_object`""" + return self._retry(head_object, container, obj) + + def get_object(self, container, obj, resp_chunk_size=None): + """Wrapper for :func:`get_object`""" + return self._retry(get_object, container, obj, + resp_chunk_size=resp_chunk_size) + + def put_object(self, container, obj, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, + headers=None): + """Wrapper for :func:`put_object`""" + return self._retry(put_object, container, obj, contents, + content_length=content_length, etag=etag, chunk_size=chunk_size, + content_type=content_type, headers=headers) + + def post_object(self, container, obj, headers): + """Wrapper for :func:`post_object`""" + return self._retry(post_object, container, obj, headers) + + def delete_object(self, container, obj): + """Wrapper for :func:`delete_object`""" + return self._retry(delete_object, container, obj) + +# End inclusion of swift.common.client +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # + + def mkdirs(path): try: makedirs(path) @@ -865,12 +863,21 @@ st_delete_help = ''' delete --all OR delete container [object] [object] ... Deletes everything in the account (with --all), or everything in a container, or a list of objects depending on the args given.'''.strip('\n') -def st_delete(options, args): + + +def st_delete(parser, args, print_queue, error_queue): + parser.add_option('-a', '--all', action='store_true', dest='yes_all', + default=False, help='Indicates that you really want to delete ' + 'everything in the account') + (options, args) = parse_args(parser, args) + args = args[1:] if (not args and not options.yes_all) or (args and options.yes_all): - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_delete_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_delete_help)) return + object_queue = Queue(10000) + def _delete_object((container, obj), conn): try: conn.delete_object(container, obj) @@ -878,13 +885,14 @@ def st_delete(options, args): path = options.yes_all and join(container, obj) or obj if path[:1] in ('/', '\\'): path = path[1:] - options.print_queue.put(path) + print_queue.put(path) except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Object %s not found' % - repr('%s/%s' % (container, obj))) + error_queue.put('Object %s not found' % + repr('%s/%s' % (container, obj))) container_queue = Queue(10000) + def _delete_container(container, conn): try: marker = '' @@ -913,11 +921,12 @@ def st_delete(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Container %s not found' % repr(container)) - url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + error_queue.put('Container %s not found' % repr(container)) + + url, token = get_auth(options.auth, options.user, options.key, + snet=options.snet) create_connection = lambda: Connection(options.auth, options.user, - options.key, preauthurl=url, - preauthtoken=token, snet=options.snet) + options.key, preauthurl=url, preauthtoken=token, snet=options.snet) object_threads = [QueueFunctionThread(object_queue, _delete_object, create_connection()) for _ in xrange(10)] for thread in object_threads: @@ -945,7 +954,7 @@ def st_delete(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Account not found') + error_queue.put('Account not found') elif len(args) == 1: conn = create_connection() _delete_container(args[0], conn) @@ -969,23 +978,39 @@ def st_delete(options, args): st_download_help = ''' download --all OR download container [object] [object] ... Downloads everything in the account (with --all), or everything in a - container, or a list of objects depending on the args given. Use - the -o [--output] option to redirect the output to a file - or if "-" then the just redirect to stdout. '''.strip('\n') -def st_download(options, args): + container, or a list of objects depending on the args given. For a single + object download, you may use the -o [--output] option to + redirect the output to a specific file or if "-" then just redirect to + stdout.'''.strip('\n') + + +def st_download(options, args, print_queue, error_queue): + parser.add_option('-a', '--all', action='store_true', dest='yes_all', + default=False, help='Indicates that you really want to download ' + 'everything in the account') + parser.add_option('-o', '--output', dest='out_file', help='For a single ' + 'file download, stream the output to an alternate location ') + (options, args) = parse_args(parser, args) + args = args[1:] + if options.out_file == '-': + options.verbose = 0 + if options.out_file and len(args) != 2: + exit('-o option only allowed for single file downloads') if (not args and not options.yes_all) or (args and options.yes_all): - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_download_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_download_help)) return + object_queue = Queue(10000) + def _download_object(queue_arg, conn): - if len(queue_arg) == 2: + if len(queue_arg) == 2: container, obj = queue_arg out_file = None elif len(queue_arg) == 3: container, obj, out_file = queue_arg else: - raise Exception("Invalid queue_arg length of %s" % len(queue_arg)) + raise Exception("Invalid queue_arg length of %s" % len(queue_arg)) try: headers, body = \ conn.get_object(container, obj, resp_chunk_size=65536) @@ -1015,29 +1040,30 @@ def st_download(options, args): fp = open(path, 'wb') read_length = 0 md5sum = md5() - for chunk in body : + for chunk in body: fp.write(chunk) read_length += len(chunk) md5sum.update(chunk) fp.close() if md5sum.hexdigest() != etag: - options.error_queue.put('%s: md5sum != etag, %s != %s' % - (path, md5sum.hexdigest(), etag)) + error_queue.put('%s: md5sum != etag, %s != %s' % + (path, md5sum.hexdigest(), etag)) if read_length != content_length: - options.error_queue.put( - '%s: read_length != content_length, %d != %d' % - (path, read_length, content_length)) + error_queue.put('%s: read_length != content_length, %d != %d' % + (path, read_length, content_length)) if 'x-object-meta-mtime' in headers and not options.out_file: mtime = float(headers['x-object-meta-mtime']) utime(path, (mtime, mtime)) if options.verbose: - options.print_queue.put(path) + print_queue.put(path) except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Object %s not found' % - repr('%s/%s' % (container, obj))) + error_queue.put('Object %s not found' % + repr('%s/%s' % (container, obj))) + container_queue = Queue(10000) + def _download_container(container, conn): try: marker = '' @@ -1052,11 +1078,12 @@ def st_download(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Container %s not found' % repr(container)) - url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + error_queue.put('Container %s not found' % repr(container)) + + url, token = get_auth(options.auth, options.user, options.key, + snet=options.snet) create_connection = lambda: Connection(options.auth, options.user, - options.key, preauthurl=url, - preauthtoken=token, snet=options.snet) + options.key, preauthurl=url, preauthtoken=token, snet=options.snet) object_threads = [QueueFunctionThread(object_queue, _download_object, create_connection()) for _ in xrange(10)] for thread in object_threads: @@ -1080,7 +1107,7 @@ def st_download(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Account not found') + error_queue.put('Account not found') elif len(args) == 1: _download_container(args[0], create_connection()) else: @@ -1112,12 +1139,24 @@ list [options] [container] items with the given delimiter (see Cloud Files general documentation for what this means). '''.strip('\n') -def st_list(options, args): + + +def st_list(options, args, print_queue, error_queue): + parser.add_option('-p', '--prefix', dest='prefix', help='Will only list ' + 'items beginning with the prefix') + parser.add_option('-d', '--delimiter', dest='delimiter', help='Will roll ' + 'up items with the given delimiter (see Cloud Files general ' + 'documentation for what this means)') + (options, args) = parse_args(parser, args) + args = args[1:] + if options.delimiter and not args: + exit('-d option only allowed for container listings') if len(args) > 1: - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_list_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_list_help)) return - conn = Connection(options.auth, options.user, options.key, snet=options.snet) + conn = Connection(options.auth, options.user, options.key, + snet=options.snet) try: marker = '' while True: @@ -1130,35 +1169,39 @@ def st_list(options, args): if not items: break for item in items: - options.print_queue.put(item.get('name', item.get('subdir'))) + print_queue.put(item.get('name', item.get('subdir'))) marker = items[-1].get('name', items[-1].get('subdir')) except ClientException, err: if err.http_status != 404: raise if not args: - options.error_queue.put('Account not found') + error_queue.put('Account not found') else: - options.error_queue.put('Container %s not found' % repr(args[0])) + error_queue.put('Container %s not found' % repr(args[0])) st_stat_help = ''' stat [container] [object] Displays information for the account, container, or object depending on the args given (if any).'''.strip('\n') -def st_stat(options, args): + + +def st_stat(options, args, print_queue, error_queue): + (options, args) = parse_args(parser, args) + args = args[1:] conn = Connection(options.auth, options.user, options.key) if not args: try: headers = conn.head_account() if options.verbose > 1: - options.print_queue.put(''' + print_queue.put(''' StorageURL: %s Auth Token: %s '''.strip('\n') % (conn.url, conn.token)) container_count = int(headers.get('x-account-container-count', 0)) object_count = int(headers.get('x-account-object-count', 0)) bytes_used = int(headers.get('x-account-bytes-used', 0)) - options.print_queue.put(''' + print_queue.put(''' Account: %s Containers: %d Objects: %d @@ -1166,24 +1209,24 @@ Containers: %d object_count, bytes_used)) for key, value in headers.items(): if key.startswith('x-account-meta-'): - options.print_queue.put('%10s: %s' % ('Meta %s' % + print_queue.put('%10s: %s' % ('Meta %s' % key[len('x-account-meta-'):].title(), value)) for key, value in headers.items(): if not key.startswith('x-account-meta-') and key not in ( 'content-length', 'date', 'x-account-container-count', 'x-account-object-count', 'x-account-bytes-used'): - options.print_queue.put( + print_queue.put( '%10s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Account not found') + error_queue.put('Account not found') elif len(args) == 1: try: headers = conn.head_container(args[0]) object_count = int(headers.get('x-container-object-count', 0)) bytes_used = int(headers.get('x-container-bytes-used', 0)) - options.print_queue.put(''' + print_queue.put(''' Account: %s Container: %s Objects: %d @@ -1195,23 +1238,23 @@ Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], headers.get('x-container-write', ''))) for key, value in headers.items(): if key.startswith('x-container-meta-'): - options.print_queue.put('%9s: %s' % ('Meta %s' % + print_queue.put('%9s: %s' % ('Meta %s' % key[len('x-container-meta-'):].title(), value)) for key, value in headers.items(): if not key.startswith('x-container-meta-') and key not in ( 'content-length', 'date', 'x-container-object-count', 'x-container-bytes-used', 'x-container-read', 'x-container-write'): - options.print_queue.put( + print_queue.put( '%9s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Container %s not found' % repr(args[0])) + error_queue.put('Container %s not found' % repr(args[0])) elif len(args) == 2: try: headers = conn.head_object(args[0], args[1]) - options.print_queue.put(''' + print_queue.put(''' Account: %s Container: %s Object: %s @@ -1225,22 +1268,22 @@ Content Length: %s headers.get('etag'))) for key, value in headers.items(): if key.startswith('x-object-meta-'): - options.print_queue.put('%14s: %s' % ('Meta %s' % + print_queue.put('%14s: %s' % ('Meta %s' % key[len('x-object-meta-'):].title(), value)) for key, value in headers.items(): if not key.startswith('x-object-meta-') and key not in ( 'content-type', 'content-length', 'last-modified', 'etag', 'date'): - options.print_queue.put( + print_queue.put( '%14s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Object %s not found' % - repr('%s/%s' % (args[0], args[1]))) + error_queue.put('Object %s not found' % + repr('%s/%s' % (args[0], args[1]))) else: - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_stat_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_stat_help)) st_post_help = ''' @@ -1252,7 +1295,22 @@ post [options] [container] [object] or --meta option is allowed on all and used to define the user meta data items to set in the form Name:Value. This option can be repeated. Example: post -m Color:Blue -m Size:Large'''.strip('\n') -def st_post(options, args): + + +def st_post(options, args, print_queue, error_queue): + parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the ' + 'Read ACL for containers. Quick summary of ACL syntax: .r:*, ' + '.r:-.example.com, .r:www.example.com, account1, account2:user2') + parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the ' + 'Write ACL for containers. Quick summary of ACL syntax: account1, ' + 'account2:user2') + parser.add_option('-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item with the syntax name:value. This option ' + 'may be repeated. Example: -m Color:Blue -m Size:Large') + (options, args) = parse_args(parser, args) + args = args[1:] + if (options.read_acl or options.write_acl) and not args: + exit('-r and -w options only allowed for containers') conn = Connection(options.auth, options.user, options.key) if not args: headers = {} @@ -1265,7 +1323,7 @@ def st_post(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Account not found') + error_queue.put('Account not found') elif len(args) == 1: headers = {} for item in options.meta: @@ -1293,11 +1351,11 @@ def st_post(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Object %s not found' % - repr('%s/%s' % (args[0], args[1]))) + error_queue.put('Object %s not found' % + repr('%s/%s' % (args[0], args[1]))) else: - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_post_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_post_help)) st_upload_help = ''' @@ -1305,12 +1363,21 @@ upload [options] container file_or_directory [file_or_directory] [...] Uploads to the given container the files and directories specified by the remaining args. -c or --changed is an option that will only upload files that have changed since the last upload.'''.strip('\n') -def st_upload(options, args): + + +def st_upload(options, args, print_queue, error_queue): + parser.add_option('-c', '--changed', action='store_true', dest='changed', + default=False, help='Will only upload files that have changed since ' + 'the last upload') + (options, args) = parse_args(parser, args) + args = args[1:] if len(args) < 2: - options.error_queue.put('Usage: %s [options] %s' % - (basename(argv[0]), st_upload_help)) + error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_upload_help)) return + file_queue = Queue(10000) + def _upload_file((path, dir_marker), conn): try: obj = path @@ -1352,11 +1419,12 @@ def st_upload(options, args): content_length=getsize(path), headers=put_headers) if options.verbose: - options.print_queue.put(obj) + print_queue.put(obj) except OSError, err: if err.errno != ENOENT: raise - options.error_queue.put('Local file %s not found' % repr(path)) + error_queue.put('Local file %s not found' % repr(path)) + def _upload_dir(path): names = listdir(path) if not names: @@ -1368,10 +1436,11 @@ def st_upload(options, args): _upload_dir(subpath) else: file_queue.put((subpath, False)) # dir_marker = False - url, token = get_auth(options.auth, options.user, options.key, snet=options.snet) + + url, token = get_auth(options.auth, options.user, options.key, + snet=options.snet) create_connection = lambda: Connection(options.auth, options.user, - options.key, preauthurl=url, - preauthtoken=token, snet=options.snet) + options.key, preauthurl=url, preauthtoken=token, snet=options.snet) file_threads = [QueueFunctionThread(file_queue, _upload_file, create_connection()) for _ in xrange(10)] for thread in file_threads: @@ -1400,12 +1469,24 @@ def st_upload(options, args): except ClientException, err: if err.http_status != 404: raise - options.error_queue.put('Account not found') + error_queue.put('Account not found') + + +def parse_args(parser, args, enforce_requires=True): + if not args: + args = ['-h'] + (options, args) = parser.parse_args(args) + if enforce_requires and \ + not (options.auth and options.user and options.key): + exit(''' +Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or +overridden with -A, -U, or -K.'''.strip('\n')) + return options, args if __name__ == '__main__': parser = OptionParser(version='%prog 1.0', usage=''' -Usage: %%prog [options] [args] +Usage: %%prog [options] [args] Commands: %(st_stat_help)s @@ -1424,55 +1505,18 @@ Example: default=1, help='Print more info') parser.add_option('-q', '--quiet', action='store_const', dest='verbose', const=0, default=1, help='Suppress status output') - parser.add_option('-a', '--all', action='store_true', dest='yes_all', - default=False, help='Indicate that you really want the ' - 'whole account for commands that require --all in such ' - 'a case') - parser.add_option('-c', '--changed', action='store_true', dest='changed', - default=False, help='For the upload command: will ' - 'only upload files that have changed since the last ' - 'upload') - parser.add_option('-p', '--prefix', dest='prefix', - help='For the list command: will only list items ' - 'beginning with the prefix') - parser.add_option('-d', '--delimiter', dest='delimiter', - help='For the list command on containers: will roll up ' - 'items with the given delimiter (see Cloud Files ' - 'general documentation for what this means).') - parser.add_option('-r', '--read-acl', dest='read_acl', - help='Sets the Read ACL with post container commands. ' - 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' - '.r:www.example.com, account1, account2:user2') - parser.add_option('-w', '--write-acl', dest='write_acl', - help='Sets the Write ACL with post container commands. ' - 'Quick summary of ACL syntax: account1, account2:user2') - parser.add_option('-m', '--meta', action='append', dest='meta', default=[], - help='Sets a meta data item of the syntax name:value ' - 'for use with post commands. This option may be ' - 'repeated. Example: -m Color:Blue -m Size:Large') parser.add_option('-A', '--auth', dest='auth', + default=environ.get('ST_AUTH'), help='URL for obtaining an auth token') parser.add_option('-U', '--user', dest='user', + default=environ.get('ST_USER'), help='User name for obtaining an auth token') parser.add_option('-K', '--key', dest='key', + default=environ.get('ST_KEY'), help='Key for obtaining an auth token') - parser.add_option('-o', '--output', dest='out_file', - help='For a single file download stream the output other location ') - args = argv[1:] - if not args: - args.append('-h') - (options, args) = parser.parse_args(args) - if options.out_file == '-': - options.verbose = 0 - - required_help = ''' -Requires ST_AUTH, ST_USER, and ST_KEY environment variables be set or -overridden with -A, -U, or -K.'''.strip('\n') - for attr in ('auth', 'user', 'key'): - if not getattr(options, attr, None): - setattr(options, attr, environ.get('ST_%s' % attr.upper())) - if not getattr(options, attr, None): - exit(required_help) + parser.disable_interspersed_args() + (options, args) = parse_args(parser, argv[1:], enforce_requires=False) + parser.enable_interspersed_args() commands = ('delete', 'download', 'list', 'post', 'stat', 'upload') if not args or args[0] not in commands: @@ -1481,30 +1525,36 @@ overridden with -A, -U, or -K.'''.strip('\n') exit('no such command: %s' % args[0]) exit() - options.print_queue = Queue(10000) + print_queue = Queue(10000) + def _print(item): if isinstance(item, unicode): item = item.encode('utf8') print item - print_thread = QueueFunctionThread(options.print_queue, _print) + + print_thread = QueueFunctionThread(print_queue, _print) print_thread.start() - options.error_queue = Queue(10000) + error_queue = Queue(10000) + def _error(item): if isinstance(item, unicode): item = item.encode('utf8') - print >>stderr, item - error_thread = QueueFunctionThread(options.error_queue, _error) + print >> stderr, item + + error_thread = QueueFunctionThread(error_queue, _error) error_thread.start() try: - globals()['st_%s' % args[0]](options, args[1:]) - while not options.print_queue.empty(): + parser.usage = globals()['st_%s_help' % args[0]] + globals()['st_%s' % args[0]](parser, argv[1:], print_queue, + error_queue) + while not print_queue.empty(): sleep(0.01) print_thread.abort = True while print_thread.isAlive(): print_thread.join(0.01) - while not options.error_queue.empty(): + while not error_queue.empty(): sleep(0.01) error_thread.abort = True while error_thread.isAlive(): diff --git a/swift/common/client.py b/swift/common/client.py index 06c3dab067..b89d06aa66 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -29,8 +29,12 @@ try: except: from time import sleep -from swift.common.bufferedhttp \ - import BufferedHTTPConnection as HTTPConnection +try: + from swift.common.bufferedhttp \ + import BufferedHTTPConnection as HTTPConnection +except: + from httplib import HTTPConnection + def quote(value, safe='/'): """