diff --git a/bin/st b/bin/st index ede5c985de..51a7637881 100755 --- a/bin/st +++ b/bin/st @@ -154,19 +154,26 @@ except: def get_auth(url, user, key, snet=False): """ - Get authentication credentials + Get authentication/authorization credentials. - :param url: authentication URL - :param user: user to auth as - :param key: key or passowrd for auth - :param snet: use SERVICENET internal network default is False - :returns: tuple of (storage URL, storage token, auth token) + The snet parameter is used for Rackspace's ServiceNet internal network + implementation. In this function, it simply adds *snet-* to the beginning + of the host name for the returned storage URL. With Rackspace Cloud Files, + use of this network path causes no bandwidth charges but requires the + client to be running on Rackspace's ServiceNet network. + + :param url: authentication/authorization URL + :param user: user to authenticate as + :param key: key or password for authorization + :param snet: use SERVICENET internal network (see above), default is False + :returns: tuple of (storage URL, auth token) :raises ClientException: HTTP GET request to auth URL failed """ parsed, conn = http_connection(url) conn.request('GET', parsed.path, '', {'X-Auth-User': user, 'X-Auth-Key': key}) resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException('Auth GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, @@ -175,7 +182,7 @@ except: url = resp.getheader('x-storage-url') if snet: parsed = list(urlparse(url)) - # Second item in the list is the netloc + # Second item in the list is the netloc parsed[1] = 'snet-' + parsed[1] url = urlunparse(parsed) return url, resp.getheader('x-storage-token', @@ -196,18 +203,21 @@ except: conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of accounts + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_account(url, token, marker, limit, prefix, http_conn) + rv = get_account(url, token, marker, limit, prefix, http_conn) + listing = rv[1] while listing: - rv.extend(listing) marker = listing[-1]['name'] - listing = get_account(url, token, marker, limit, prefix, http_conn) + listing = \ + get_account(url, token, marker, limit, prefix, http_conn)[1] + if listing: + rv.extend(listing) return rv parsed, conn = http_conn qs = 'format=json' @@ -220,6 +230,9 @@ except: conn.request('GET', '%s?%s' % (parsed.path, qs), '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException('Account GET failed', http_scheme=parsed.scheme, @@ -228,8 +241,8 @@ except: http_reason=resp.reason) if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_account(url, token, http_conn=None): @@ -240,7 +253,8 @@ except: :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (container count, object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -249,14 +263,42 @@ except: parsed, conn = http_connection(url) conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-account-container-count', 0)), \ - int(resp.getheader('x-account-object-count', 0)), \ - int(resp.getheader('x-account-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + + def post_account(url, token, headers, http_conn=None): + """ + Update an account's metadata. + + :param url: storage URL + :param token: auth token + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + headers['X-Auth-Token'] = token + conn.request('POST', parsed.path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) def get_container(url, token, container, marker=None, limit=None, @@ -276,23 +318,25 @@ except: conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) + rv = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + listing = rv[1] while listing: - rv.extend(listing) if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn) + prefix, delimiter, http_conn)[1] + if listing: + rv[1].extend(listing) return rv parsed, conn = http_conn path = '%s/%s' % (parsed.path, quote(container)) @@ -313,10 +357,13 @@ except: http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_query=qs, http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_container(url, token, container, http_conn=None): @@ -328,7 +375,8 @@ except: :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -344,17 +392,20 @@ except: http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count', 0)), \ - int(resp.getheader('x-container-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers - def put_container(url, token, container, http_conn=None): + def put_container(url, token, container, headers=None, http_conn=None): """ Create a container :param url: storage URL :param token: auth token :param container: container name to create + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP PUT request failed @@ -364,7 +415,10 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) - conn.request('PUT', path, '', {'X-Auth-Token': token}) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + conn.request('PUT', path, '', headers) resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: @@ -374,6 +428,34 @@ except: http_reason=resp.reason) + def post_container(url, token, container, headers, http_conn=None): + """ + Update a container's metadata. + + :param url: storage URL + :param token: auth token + :param container: container name to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + def delete_container(url, token, container, http_conn=None): """ Delete a container @@ -411,8 +493,12 @@ except: :param name: object name to get :param http_conn: HTTP connection object (If None, it will create the conn object) - :param resp_chunk_size: if defined, chunk size of data to read - :returns: a list of objects + :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If + you specify a resp_chunk_size you must fully read + the object's contents before making another + request. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if http_conn: @@ -427,10 +513,6 @@ except: raise ClientException('Object GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) if resp_chunk_size: def _object_body(): @@ -441,12 +523,10 @@ except: object_body = _object_body() else: object_body = resp.read() - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata, \ - object_body + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def head_object(url, token, container, name, http_conn=None): @@ -459,8 +539,8 @@ except: :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (content type, content length, last modfied, etag, - dictionary of metadata) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -475,20 +555,15 @@ except: raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers - def put_object(url, token, container, name, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None, http_conn=None): + def put_object(url, token, container, name, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, headers=None, + http_conn=None): """ Put an object @@ -496,12 +571,12 @@ except: :param token: auth token :param container: container name that the object is in :param name: object name to put - :param contents: file like object to read object data from - :param metadata: dictionary of object metadata + :param contents: a string or a file like object to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param chunk_size: chunk size of data to write :param content_type: value to send as content-type header + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :returns: etag from server response @@ -512,9 +587,9 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if not headers: + headers = {} + headers['X-Auth-Token'] = token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -550,15 +625,15 @@ except: return resp.getheader('etag').strip('"') - def post_object(url, token, container, name, metadata, http_conn=None): + def post_object(url, token, container, name, headers, http_conn=None): """ - Change object metadata + Update object metadata :param url: storage URL :param token: auth token :param container: container name that the object is in - :param name: object name to change - :param metadata: dictionary of object metadata + :param name: name of the object to update + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP POST request failed @@ -568,9 +643,7 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + headers['X-Auth-Token'] = token conn.request('POST', path, '', headers) resp = conn.getresponse() resp.read() @@ -620,7 +693,7 @@ except: :param preauthurl: storage URL (if you have already authenticated) :param preauthtoken: authentication token (if you have already authenticated) - :param snet: use SERVICENET internal network default is False + :param snet: use SERVICENET internal network default is False """ self.authurl = authurl self.user = user @@ -632,20 +705,24 @@ except: self.attempts = 0 self.snet = snet + def get_auth(self): + return get_auth(self.authurl, self.user, self.key, snet=self.snet) + + def http_connection(self): + return http_connection(self.url) + def _retry(self, func, *args, **kwargs): - kwargs['http_conn'] = self.http_conn self.attempts = 0 backoff = 1 while self.attempts <= self.retries: self.attempts += 1 try: if not self.url or not self.token: - self.url, self.token = \ - get_auth(self.authurl, self.user, self.key, snet=self.snet) + self.url, self.token = self.get_auth() self.http_conn = None if not self.http_conn: - self.http_conn = http_connection(self.url) - kwargs['http_conn'] = self.http_conn + self.http_conn = self.http_connection() + kwargs['http_conn'] = self.http_conn rv = func(self.url, self.token, *args, **kwargs) return rv except (socket.error, HTTPException): @@ -667,63 +744,71 @@ except: backoff *= 2 def head_account(self): - """Wrapper for head_account""" + """Wrapper for :func:`head_account`""" return self._retry(head_account) def get_account(self, marker=None, limit=None, prefix=None, full_listing=False): - """Wrapper for get_account""" - # TODO: With full_listing=True this will restart the entire listing - # with each retry. Need to make a better version that just retries - # where it left off. + """Wrapper for :func:`get_account`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. return self._retry(get_account, marker=marker, limit=limit, prefix=prefix, full_listing=full_listing) + def post_account(self, headers): + """Wrapper for :func:`post_account`""" + return self._retry(post_account, headers) + def head_container(self, container): - """Wrapper for head_container""" + """Wrapper for :func:`head_container`""" return self._retry(head_container, container) def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, full_listing=False): - """Wrapper for get_container""" - # TODO: With full_listing=True this will restart the entire listing - # with each retry. Need to make a better version that just retries - # where it left off. + """Wrapper for :func:`get_container`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. return self._retry(get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, full_listing=full_listing) - def put_container(self, container): - """Wrapper for put_container""" - return self._retry(put_container, container) + def put_container(self, container, headers=None): + """Wrapper for :func:`put_container`""" + return self._retry(put_container, container, headers=headers) + + def post_container(self, container, headers): + """Wrapper for :func:`post_container`""" + return self._retry(post_container, container, headers) def delete_container(self, container): - """Wrapper for delete_container""" + """Wrapper for :func:`delete_container`""" return self._retry(delete_container, container) def head_object(self, container, obj): - """Wrapper for head_object""" + """Wrapper for :func:`head_object`""" return self._retry(head_object, container, obj) def get_object(self, container, obj, resp_chunk_size=None): - """Wrapper for get_object""" + """Wrapper for :func:`get_object`""" return self._retry(get_object, container, obj, resp_chunk_size=resp_chunk_size) - def put_object(self, container, obj, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None): - """Wrapper for put_object""" + def put_object(self, container, obj, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, + headers=None): + """Wrapper for :func:`put_object`""" return self._retry(put_object, container, obj, contents, - metadata=metadata, content_length=content_length, etag=etag, - chunk_size=chunk_size, content_type=content_type) + content_length=content_length, etag=etag, chunk_size=chunk_size, + content_type=content_type, headers=headers) - def post_object(self, container, obj, metadata): - """Wrapper for post_object""" - return self._retry(post_object, container, obj, metadata) + def post_object(self, container, obj, headers): + """Wrapper for :func:`post_object`""" + return self._retry(post_object, container, obj, headers) def delete_object(self, container, obj): - """Wrapper for delete_object""" + """Wrapper for :func:`delete_object`""" return self._retry(delete_object, container, obj) # End inclusion of swift.common.client @@ -805,7 +890,7 @@ def st_delete(options, args): marker = '' while True: objects = [o['name'] for o in - conn.get_container(container, marker=marker)] + conn.get_container(container, marker=marker)[1]] if not objects: break for obj in objects: @@ -847,7 +932,7 @@ def st_delete(options, args): marker = '' while True: containers = \ - [c['name'] for c in conn.get_account(marker=marker)] + [c['name'] for c in conn.get_account(marker=marker)[1]] if not containers: break for container in containers: @@ -893,8 +978,11 @@ def st_download(options, args): object_queue = Queue(10000) def _download_object((container, obj), conn): try: - content_type, content_length, _, etag, metadata, body = \ + headers, body = \ conn.get_object(container, obj, resp_chunk_size=65536) + content_type = headers.get('content-type') + content_length = int(headers.get('content-length')) + etag = headers.get('etag') path = options.yes_all and join(container, obj) or obj if path[:1] in ('/', '\\'): path = path[1:] @@ -925,8 +1013,8 @@ def st_download(options, args): options.error_queue.put( '%s: read_length != content_length, %d != %d' % (path, read_length, content_length)) - if 'mtime' in metadata: - mtime = float(metadata['mtime']) + if 'x-object-meta-mtime' in headers: + mtime = float(headers['x-object-meta-mtime']) utime(path, (mtime, mtime)) if options.verbose: options.print_queue.put(path) @@ -941,7 +1029,7 @@ def st_download(options, args): marker = '' while True: objects = [o['name'] for o in - conn.get_container(container, marker=marker)] + conn.get_container(container, marker=marker)[1]] if not objects: break for obj in objects: @@ -969,7 +1057,7 @@ def st_download(options, args): marker = '' while True: containers = [c['name'] - for c in conn.get_account(marker=marker)] + for c in conn.get_account(marker=marker)[1]] if not containers: break for container in containers: @@ -1016,10 +1104,11 @@ def st_list(options, args): marker = '' while True: if not args: - items = conn.get_account(marker=marker, prefix=options.prefix) + items = \ + conn.get_account(marker=marker, prefix=options.prefix)[1] else: items = conn.get_container(args[0], marker=marker, - prefix=options.prefix, delimiter=options.delimiter) + prefix=options.prefix, delimiter=options.delimiter)[1] if not items: break for item in items: @@ -1042,46 +1131,85 @@ def st_stat(options, args): conn = Connection(options.auth, options.user, options.key) if not args: try: - container_count, object_count, bytes_used = conn.head_account() + headers = conn.head_account() + container_count = int(headers.get('x-account-container-count', 0)) + object_count = int(headers.get('x-account-object-count', 0)) + bytes_used = int(headers.get('x-account-bytes-used', 0)) options.print_queue.put(''' Account: %s Containers: %d Objects: %d Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count, object_count, bytes_used)) + for key, value in headers.items(): + if key.startswith('x-account-meta-'): + options.print_queue.put('%10s: %s' % ('Meta %s' % + key[len('x-account-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-account-meta-') and key not in ( + 'content-length', 'date', 'x-account-container-count', + 'x-account-object-count', 'x-account-bytes-used'): + options.print_queue.put( + '%10s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise options.error_queue.put('Account not found') elif len(args) == 1: try: - object_count, bytes_used = conn.head_container(args[0]) + headers = conn.head_container(args[0]) + object_count = int(headers.get('x-container-object-count', 0)) + bytes_used = int(headers.get('x-container-bytes-used', 0)) options.print_queue.put(''' Account: %s Container: %s Objects: %d - Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], - object_count, bytes_used)) + Bytes: %d + Read ACL: %s +Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], + object_count, bytes_used, + headers.get('x-container-read', ''), + headers.get('x-container-write', ''))) + for key, value in headers.items(): + if key.startswith('x-container-meta-'): + options.print_queue.put('%9s: %s' % ('Meta %s' % + key[len('x-container-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-container-meta-') and key not in ( + 'content-length', 'date', 'x-container-object-count', + 'x-container-bytes-used', 'x-container-read', + 'x-container-write'): + options.print_queue.put( + '%9s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise options.error_queue.put('Container %s not found' % repr(args[0])) elif len(args) == 2: try: - content_type, content_length, last_modified, etag, metadata = \ - conn.head_object(args[0], args[1]) + headers = conn.head_object(args[0], args[1]) options.print_queue.put(''' Account: %s Container: %s Object: %s Content Type: %s -Content Length: %d +Content Length: %s Last Modified: %s ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], - args[1], content_type, content_length, - last_modified, etag)) - for key, value in metadata.items(): - options.print_queue.put('%14s: %s' % ('Meta %s' % key, value)) + args[1], headers.get('content-type'), + headers.get('content-length'), + headers.get('last-modified'), + headers.get('etag'))) + for key, value in headers.items(): + if key.startswith('x-object-meta-'): + options.print_queue.put('%14s: %s' % ('Meta %s' % + key[len('x-object-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-object-meta-') and key not in ( + 'content-type', 'content-length', 'last-modified', + 'etag', 'date'): + options.print_queue.put( + '%14s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise @@ -1092,6 +1220,63 @@ Content Length: %d (basename(argv[0]), st_stat_help)) +st_post_help = ''' +post [options] [container] [object] + Updates meta information for the account, container, or object depending on + the args given. If the container is not found, it will be created + automatically; but this is not true for accounts and objects. Containers + also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m + or --meta option is allowed on all and used to define the user meta data + items to set in the form Name:Value. This option can be repeated. Example: + post -m Color:Blue -m Size:Large'''.strip('\n') +def st_post(options, args): + conn = Connection(options.auth, options.user, options.key) + if not args: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Account-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + try: + conn.post_account(headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + elif len(args) == 1: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Container-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + if options.read_acl is not None: + headers['X-Container-Read'] = options.read_acl + if options.write_acl is not None: + headers['X-Container-Write'] = options.write_acl + try: + conn.post_container(args[0], headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + conn.put_container(args[0], headers=headers) + elif len(args) == 2: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Object-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + try: + conn.post_object(args[0], args[1], headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Object %s not found' % + repr('%s/%s' % (args[0], args[1]))) + else: + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_post_help)) + + st_upload_help = ''' upload [options] container file_or_directory [file_or_directory] [...] Uploads to the given container the files and directories specified by the @@ -1108,35 +1293,41 @@ def st_upload(options, args): obj = path if obj.startswith('./') or obj.startswith('.\\'): obj = obj[2:] - metadata = {'mtime': str(getmtime(path))} + put_headers = {'x-object-meta-mtime': str(getmtime(path))} if dir_marker: if options.changed: try: - ct, cl, lm, et, md = conn.head_object(args[0], obj) + headers = conn.head_object(args[0], obj) + ct = headers.get('content-type') + cl = int(headers.get('content-length')) + et = headers.get('etag') + mt = headers.get('x-object-meta-mtime') if ct.split(';', 1)[0] == 'text/directory' and \ cl == 0 and \ et == 'd41d8cd98f00b204e9800998ecf8427e' and \ - md.get('mtime') == metadata['mtime']: + mt == put_headers['x-object-meta-mtime']: return except ClientException, err: if err.http_status != 404: raise conn.put_object(args[0], obj, '', content_length=0, content_type='text/directory', - metadata=metadata) + headers=put_headers) else: if options.changed: try: - ct, cl, lm, et, md = conn.head_object(args[0], obj) + headers = conn.head_object(args[0], obj) + cl = int(headers.get('content-length')) + mt = headers.get('x-object-meta-mtime') if cl == getsize(path) and \ - md.get('mtime') == metadata['mtime']: + mt == put_headers['x-object-meta-mtime']: return except ClientException, err: if err.http_status != 404: raise conn.put_object(args[0], obj, open(path, 'rb'), content_length=getsize(path), - metadata=metadata) + headers=put_headers) if options.verbose: options.print_queue.put(obj) except OSError, err: @@ -1163,8 +1354,15 @@ def st_upload(options, args): for thread in file_threads: thread.start() conn = create_connection() + # Try to create the container, just in case it doesn't exist. If this + # fails, it might just be because the user doesn't have container PUT + # permissions, so we'll ignore any error. If there's really a problem, + # it'll surface on the first object PUT. try: conn.put_container(args[0]) + except: + pass + try: for arg in args[1:]: if isdir(arg): _upload_dir(arg) @@ -1190,6 +1388,7 @@ Commands: %(st_stat_help)s %(st_list_help)s %(st_upload_help)s + %(st_post_help)s %(st_download_help)s %(st_delete_help)s @@ -1215,6 +1414,17 @@ Example: help='For the list command on containers: will roll up ' 'items with the given delimiter (see Cloud Files ' 'general documentation for what this means).') + parser.add_option('-r', '--read-acl', dest='read_acl', + help='Sets the Read ACL with post container commands. ' + 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' + '.r:www.example.com, account1, account2:user2') + parser.add_option('-w', '--write-acl', dest='write_acl', + help='Sets the Write ACL with post container commands. ' + 'Quick summary of ACL syntax: account1, account2:user2') + parser.add_option('-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item of the syntax name:value ' + 'for use with post commands. This option may be ' + 'repeated. Example: -m Color:Blue -m Size:Large') parser.add_option('-A', '--auth', dest='auth', help='URL for obtaining an auth token') parser.add_option('-U', '--user', dest='user', @@ -1235,7 +1445,7 @@ overridden with -A, -U, or -K.'''.strip('\n') if not getattr(options, attr, None): exit(required_help) - commands = ('delete', 'download', 'list', 'stat', 'upload') + commands = ('delete', 'download', 'list', 'post', 'stat', 'upload') if not args or args[0] not in commands: parser.print_usage() if args: diff --git a/bin/swift-auth-add-user b/bin/swift-auth-add-user new file mode 100755 index 0000000000..0f7eb3c3a2 --- /dev/null +++ b/bin/swift-auth-add-user @@ -0,0 +1,59 @@ +#!/usr/bin/python +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ConfigParser import ConfigParser +from optparse import OptionParser +from os.path import basename +from sys import argv, exit + +from swift.common.bufferedhttp import http_connect_raw as http_connect + + +if __name__ == '__main__': + default_conf = '/etc/swift/auth-server.conf' + parser = OptionParser( + usage='Usage: %prog [options] ') + parser.add_option('-c', '--conf', dest='conf', default=default_conf, + help='Configuration file to determine how to connect to the local ' + 'auth server (default: %s).' % default_conf) + parser.add_option('-a', '--admin', dest='admin', action='store_true', + default=False, help='Give the user administrator access; otherwise ' + 'the user will only have access to containers specifically allowed ' + 'with ACLs.') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + if len(args) != 3: + parser.parse_args(['-h']) + account, user, password = args + c = ConfigParser() + if not c.read(options.conf): + exit('Unable to read conf file: %s' % options.conf) + conf = dict(c.items('app:auth-server')) + host = conf.get('bind_ip', '127.0.0.1') + port = int(conf.get('bind_port', 11000)) + ssl = conf.get('cert_file') is not None + path = '/account/%s/%s' % (account, user) + headers = {'X-Auth-User-Key': password} + if options.admin: + headers['X-Auth-User-Admin'] = 'true' + conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl) + resp = conn.getresponse() + if resp.status == 204: + print resp.getheader('x-storage-url') + else: + print 'Update failed: %s %s' % (resp.status, resp.reason) diff --git a/bin/swift-auth-create-account b/bin/swift-auth-create-account deleted file mode 100755 index bab2d3daea..0000000000 --- a/bin/swift-auth-create-account +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ConfigParser import ConfigParser -from sys import argv, exit - -from swift.common.bufferedhttp import http_connect_raw as http_connect - - -if __name__ == '__main__': - f = '/etc/swift/auth-server.conf' - if len(argv) == 5: - f = argv[4] - elif len(argv) != 4: - exit('Syntax: %s [conf_file]' % - argv[0]) - new_account = argv[1] - new_user = argv[2] - new_password = argv[3] - c = ConfigParser() - if not c.read(f): - exit('Unable to read conf file: %s' % f) - conf = dict(c.items('app:auth-server')) - host = conf.get('bind_ip', '127.0.0.1') - port = int(conf.get('bind_port', 11000)) - ssl = conf.get('cert_file') is not None - path = '/account/%s/%s' % (new_account, new_user) - conn = http_connect(host, port, 'PUT', path, {'x-auth-key':new_password}, - ssl=ssl) - resp = conn.getresponse() - if resp.status == 204: - print resp.getheader('x-storage-url') - else: - print 'Account creation failed. (%d)' % resp.status diff --git a/bin/swift-stats-populate b/bin/swift-stats-populate index 793c8b6ce4..8ea210cb65 100755 --- a/bin/swift-stats-populate +++ b/bin/swift-stats-populate @@ -48,7 +48,8 @@ def put_object(connpool, container, obj, report): global retries_done try: with connpool.item() as conn: - conn.put_object(container, obj, obj, metadata={'stats': obj}) + conn.put_object(container, obj, obj, + headers={'x-object-meta-stats': obj}) retries_done += conn.attempts - 1 if report: report(True) diff --git a/bin/swift-stats-report b/bin/swift-stats-report index 537a731697..3f735877cf 100755 --- a/bin/swift-stats-report +++ b/bin/swift-stats-report @@ -57,7 +57,8 @@ def get_error_log(prefix): def audit(coropool, connpool, account, container_ring, object_ring, options): begun = time() with connpool.item() as conn: - estimated_items = [conn.head_account()[0]] + estimated_items = \ + [int(conn.head_account()['x-account-container-count'])] items_completed = [0] retries_done = [0] containers_missing_replicas = {} @@ -85,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): retries_done[0] += attempts - 1 found = True if not estimated_objects: - estimated_objects = info[0] + estimated_objects = int(info['x-container-object-count']) except ClientException, err: if err.http_status not in (404, 507): error_log('Giving up on /%s/%s/%s: %s' % (part, account, @@ -130,7 +131,8 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): cmarker = '' while True: with connpool.item() as conn: - containers = [c['name'] for c in conn.get_account(marker=cmarker)] + containers = \ + [c['name'] for c in conn.get_account(marker=cmarker)[1]] if not containers: break cmarker = containers[-1] @@ -142,7 +144,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): while True: with connpool.item() as conn: objects = [o['name'] for o in - conn.get_container(container, marker=omarker)] + conn.get_container(container, marker=omarker)[1]] if not objects: break omarker = objects[-1] @@ -183,7 +185,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring, with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_dispersion_', - full_listing=True)] + full_listing=True)[1]] containers_listed = len(containers) if not containers_listed: print >>stderr, 'No containers to query. Has stats-populate been run?' @@ -262,7 +264,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring, options): with connpool.item() as conn: try: objects = [o['name'] for o in conn.get_container(container, - prefix='stats_object_dispersion_', full_listing=True)] + prefix='stats_object_dispersion_', full_listing=True)[1]] except ClientException, err: if err.http_status != 404: raise @@ -384,7 +386,7 @@ def container_head_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def head(container): with connpool.item() as conn: @@ -425,7 +427,7 @@ def container_get_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def get(container): with connpool.item() as conn: @@ -463,7 +465,8 @@ def container_standard_listing_report(coropool, connpool, options): print 'Listing big_container', with connpool.item() as conn: try: - value = len(conn.get_container('big_container', full_listing=True)) + value = \ + len(conn.get_container('big_container', full_listing=True)[1]) except ClientException, err: if err.http_status != 404: raise @@ -486,7 +489,7 @@ def container_prefix_listing_report(coropool, connpool, options): try: for x in xrange(256): value += len(conn.get_container('big_container', - prefix=('%02x' % x), full_listing=True)) + prefix=('%02x' % x), full_listing=True)[1]) except ClientException, err: if err.http_status != 404: raise @@ -511,7 +514,7 @@ def container_prefix_delimiter_listing_report(coropool, connpool, options): try: with connpool.item() as conn: listing = conn.get_container('big_container', - marker=marker, prefix=prefix, delimiter='/') + marker=marker, prefix=prefix, delimiter='/')[1] except ClientException, err: if err.http_status != 404: raise @@ -552,7 +555,7 @@ def container_delete_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def delete(container): with connpool.item() as conn: @@ -630,7 +633,7 @@ def object_head_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def head(obj): with connpool.item() as conn: @@ -670,7 +673,7 @@ def object_get_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def get(obj): with connpool.item() as conn: @@ -710,7 +713,7 @@ def object_delete_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def delete(obj): with connpool.item() as conn: @@ -791,7 +794,7 @@ Usage: %prog [options] [conf_file] else: options.retries = int(conf.get('retries', 5)) if not options.csv_output: - csv_output = conf.get('csv_output', '/etc/swift/stats.csv') + options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv') coropool = GreenPool(size=concurrency) @@ -908,8 +911,8 @@ Usage: %prog [options] [conf_file] if options.csv_output != 'None': try: - if not os.path.exists(csv_output): - f = open(csv_output, 'wb') + if not os.path.exists(options.csv_output): + f = open(options.csv_output, 'wb') f.write('Timestamp,' 'Container Dispersion Report Time,' 'Container Dispersion Report Value,' @@ -936,7 +939,7 @@ Usage: %prog [options] [conf_file] 'Object DELETE Report Success Rate\r\n') csv = csv.writer(f) else: - csv = csv.writer(open(csv_output, 'ab')) + csv = csv.writer(open(options.csv_output, 'ab')) csv.writerow(report) except Exception, err: print >>stderr, 'Could not write CSV report:', err diff --git a/doc/source/auth.rst b/doc/source/auth.rst index dc5a65ac45..feb3be8a99 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -4,7 +4,7 @@ Developer's Authorization ************************* -.. _auth-server: +.. _auth_server: Auth Server =========== diff --git a/doc/source/development_auth.rst b/doc/source/development_auth.rst new file mode 100644 index 0000000000..bccb6e7df5 --- /dev/null +++ b/doc/source/development_auth.rst @@ -0,0 +1,484 @@ +========================== +Auth Server and Middleware +========================== + +-------------------------------------------- +Creating Your Own Auth Server and Middleware +-------------------------------------------- + +The included swift/common/middleware/auth.py is a good minimal example of how +to create auth middleware. The main points are that the auth middleware can +reject requests up front, before they ever get to the Swift Proxy application, +and afterwards when the proxy issues callbacks to verify authorization. + +It's generally good to separate the authentication and authorization +procedures. Authentication verifies that a request actually comes from who it +says it does. Authorization verifies the 'who' has access to the resource(s) +the request wants. + +Authentication is performed on the request before it ever gets to the Swift +Proxy application. The identity information is gleaned from the request, +validated in some way, and the validation information is added to the WSGI +environment as needed by the future authorization procedure. What exactly is +added to the WSGI environment is solely dependent on what the installed +authorization procedures need; the Swift Proxy application itself needs no +specific information, it just passes it along. Convention has +environ['REMOTE_USER'] set to the authenticated user string but often more +information is needed than just that. + +The included DevAuth will set the REMOTE_USER to a comma separated list of +groups the user belongs to. The first group will be the "user's group", a group +that only the user belongs to. The second group will be the "account's group", +a group that includes all users for that auth account (different than the +storage account). The third group is optional and is the storage account +string. If the user does not have admin access to the account, the third group +will be omitted. + +It is highly recommended that authentication server implementers prefix their +tokens and Swift storage accounts they create with a configurable reseller +prefix (`AUTH_` by default with the included DevAuth). This prefix will allow +deconflicting with other authentication servers that might be using the same +Swift cluster. Otherwise, the Swift cluster will have to try all the resellers +until one validates a token or all fail. + +A restriction with group names is that no group name should begin with a period +'.' as that is reserved for internal Swift use (such as the .r for referrer +designations as you'll see later). + +Example Authentication with DevAuth: + + * Token AUTH_tkabcd is given to the DevAuth middleware in a request's + X-Auth-Token header. + * The DevAuth middleware makes a validate token AUTH_tkabcd call to the + external DevAuth server. + * The external DevAuth server validates the token AUTH_tkabcd and discovers + it matches the "tester" user within the "test" account for the storage + account "AUTH_storage_xyz". + * The external DevAuth server responds with "X-Auth-Groups: + test:tester,test,AUTH_storage_xyz" + * Now this user will have full access (via authorization procedures later) + to the AUTH_storage_xyz Swift storage account and access to other storage + accounts with the same `AUTH_` reseller prefix and has an ACL specifying + at least one of those three groups returned. + +Authorization is performed through callbacks by the Swift Proxy server to the +WSGI environment's swift.authorize value, if one is set. The swift.authorize +value should simply be a function that takes a webob.Request as an argument and +returns None if access is granted or returns a callable(environ, +start_response) if access is denied. This callable is a standard WSGI callable. +Generally, you should return 403 Forbidden for requests by an authenticated +user and 401 Unauthorized for an unauthenticated request. For example, here's +an authorize function that only allows GETs (in this case you'd probably return +405 Method Not Allowed, but ignore that for the moment).:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + def authorize(req): + if req.method == 'GET': + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + +Adding the swift.authorize callback is often done by the authentication +middleware as authentication and authorization are often paired together. But, +you could create separate authorization middleware that simply sets the +callback before passing on the request. To continue our example above:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + if req.method == 'GET': + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The Swift Proxy server will call swift.authorize after some initial work, but +before truly trying to process the request. Positive authorization at this +point will cause the request to be fully processed immediately. A denial at +this point will immediately send the denial response for most operations. + +But for some operations that might be approved with more information, the +additional information will be gathered and added to the WSGI environment and +then swift.authorize will be called once more. These are called delay_denial +requests and currently include container read requests and object read and +write requests. For these requests, the read or write access control string +(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl' +attribute in the webob.Request passed to swift.authorize. + +The delay_denial procedures allow skipping possibly expensive access control +string retrievals for requests that can be approved without that information, +such as administrator or account owner requests. + +To further our example, we now will approve all requests that have the access +control string set to same value as the authenticated user string. Note that +you probably wouldn't do this exactly as the access control string represents a +list rather than a single user, but it'll suffice for this example:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + # Allow anyone to perform GET requests + if req.method == 'GET': + return None + # Allow any request where the acl equals the authenticated user + if getattr(req, 'acl', None) == req.remote_user: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The access control string has a standard format included with Swift, though +this can be overridden if desired. The standard format can be parsed with +swift.common.middleware.acl.parse_acl which converts the string into two arrays +of strings: (referrers, groups). The referrers allow comparing the request's +Referer header to control access. The groups allow comparing the +request.remote_user (or other sources of group information) to control access. +Checking referrer access can be accomplished by using the +swift.common.middleware.acl.referrer_allowed function. Checking group access is +usually a simple string comparison. + +Let's continue our example to use parse_acl and referrer_allowed. Now we'll +only allow GETs after a referrer check and any requests after a group check:: + + from swift.common.middleware.acl import parse_acl, referrer_allowed + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + if hasattr(req, 'acl'): + referrers, groups = parse_acl(req.acl) + if req.method == 'GET' and referrer_allowed(req, referrers): + return None + if req.remote_user and groups and req.remote_user in groups: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The access control strings are set with PUTs and POSTs to containers with the +X-Container-Read and X-Container-Write headers. Swift allows these strings to +be set to any value, though it's very useful to validate the strings meet the +desired format and return a useful error to the user if they don't. + +To support this validation, the Swift Proxy application will call the WSGI +environment's swift.clean_acl callback whenever one of these headers is to be +written. The callback should take a header name and value as its arguments. It +should return the cleaned value to save if valid or raise a ValueError with a +reasonable error message if not. + +There is an included swift.common.middleware.acl.clean_acl that validates the +standard Swift format. Let's improve our example by making use of that:: + + from swift.common.middleware.acl import \ + clean_acl, parse_acl, referrer_allowed + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = clean_acl + return self.app(environ, start_response) + + def authorize(self, req): + if hasattr(req, 'acl'): + referrers, groups = parse_acl(req.acl) + if req.method == 'GET' and referrer_allowed(req, referrers): + return None + if req.remote_user and groups and req.remote_user in groups: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +Now, if you want to override the format for access control strings you'll have +to provide your own clean_acl function and you'll have to do your own parsing +and authorization checking for that format. It's highly recommended you use the +standard format simply to support the widest range of external tools, but +sometimes that's less important than meeting certain ACL requirements. + + +---------------------------- +Integrating With repoze.what +---------------------------- + +Here's an example of integration with repoze.what, though honestly it just does +what the default swift/common/middleware/auth.py does in a slightly different +way. I'm no repoze.what expert by any stretch; this is just included here to +hopefully give folks a start on their own code if they want to use +repoze.what:: + + from time import time + + from eventlet.timeout import Timeout + from repoze.what.adapters import BaseSourceAdapter + from repoze.what.middleware import setup_auth + from repoze.what.predicates import in_any_group, NotAuthorizedError + from swift.common.bufferedhttp import http_connect_raw as http_connect + from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed + from swift.common.utils import cache_from_env, split_path + from webob.exc import HTTPForbidden, HTTPUnauthorized + + + class DevAuthorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = clean_acl + return self.app(environ, start_response) + + def authorize(self, req): + version, account, container, obj = split_path(req.path, 1, 4, True) + if not account: + return self.denied_response(req) + referrers, groups = parse_acl(getattr(req, 'acl', None)) + if referrer_allowed(req, referrers): + return None + try: + in_any_group(account, *groups).check_authorization(req.environ) + except NotAuthorizedError: + return self.denied_response(req) + return None + + def denied_response(self, req): + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + class DevIdentifier(object): + + def __init__(self, conf): + self.conf = conf + + def identify(self, env): + return {'token': + env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))} + + def remember(self, env, identity): + return [] + + def forget(self, env, identity): + return [] + + + class DevAuthenticator(object): + + def __init__(self, conf): + self.conf = conf + self.auth_host = conf.get('ip', '127.0.0.1') + self.auth_port = int(conf.get('port', 11000)) + self.ssl = \ + conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes') + self.timeout = int(conf.get('node_timeout', 10)) + + def authenticate(self, env, identity): + token = identity.get('token') + if not token: + return None + memcache_client = cache_from_env(env) + key = 'devauth/%s' % token + cached_auth_data = memcache_client.get(key) + if cached_auth_data: + start, expiration, user = cached_auth_data + if time() - start <= expiration: + return user + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s' % token, ssl=self.ssl) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status == 204: + expiration = float(resp.getheader('x-auth-ttl')) + user = resp.getheader('x-auth-user') + memcache_client.set(key, (time(), expiration, user), + timeout=expiration) + return user + return None + + + class DevChallenger(object): + + def __init__(self, conf): + self.conf = conf + + def challenge(self, env, status, app_headers, forget_headers): + def no_challenge(env, start_response): + start_response(str(status), []) + return [] + return no_challenge + + + class DevGroupSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevGroupSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, credentials): + return credentials['repoze.what.userid'].split(',') + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + + + class DevPermissionSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, group_name): + return set([n for (n, p) in self.sections.items() + if group_name in p]) + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return setup_auth(DevAuthorization(app, conf), + group_adapters={'all_groups': DevGroupSourceAdapter()}, + permission_adapters={'all_perms': DevPermissionSourceAdapter()}, + identifiers=[('devauth', DevIdentifier(conf))], + authenticators=[('devauth', DevAuthenticator(conf))], + challengers=[('devauth', DevChallenger(conf))]) + return auth_filter diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index fb0e303db4..b583641ab5 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -526,25 +526,17 @@ good idea what to do on other environments. #. `remakerings` #. `cd ~/swift/trunk; ./.unittests` #. `startmain` (The ``Unable to increase file descriptor limit. Running as non-root?`` warnings are expected and ok.) - #. `swift-auth-create-account test tester testing` + #. `swift-auth-add-user --admin test tester testing` #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` - #. Create `/etc/swift/func_test.conf`:: - - auth_host = 127.0.0.1 - auth_port = 11000 - auth_ssl = no - - account = test - username = tester - password = testing - - collate = C - - #. `cd ~/swift/trunk; ./.functests` - #. `cd ~/swift/trunk; ./.probetests` (Note for future reference: probe tests - will reset your environment) + #. `swift-auth-add-user --admin test2 tester2 testing2` + #. `swift-auth-add-user test tester3 testing3` + #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` + #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete + everything in the configured accounts.) + #. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your + environment as they call `resetswift` for each test.) If you plan to work on documentation (and who doesn't?!): diff --git a/doc/source/howto_cyberduck.rst b/doc/source/howto_cyberduck.rst index a372ebf1ab..be451f7d8c 100644 --- a/doc/source/howto_cyberduck.rst +++ b/doc/source/howto_cyberduck.rst @@ -107,9 +107,9 @@ Installing Swift For Use With Cyberduck cert_file = /etc/swift/cert.crt key_file = /etc/swift/cert.key -#. Use swift-auth-create-account to create a new account:: +#. Use swift-auth-add-user to create a new account and admin user:: - ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-create-account a3 b3 c3 + ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-add-user --admin a3 b3 c3 https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1/06228ccf-6d0a-4395-889e-e971e8de8781 .. note:: diff --git a/doc/source/index.rst b/doc/source/index.rst index 7e4681ca28..8760852f13 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,6 +32,7 @@ Development: development_guidelines development_saio + development_auth Deployment: diff --git a/doc/source/misc.rst b/doc/source/misc.rst index bae61a699a..cfd188e65a 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -42,6 +42,15 @@ Auth :members: :show-inheritance: +.. _acls: + +ACLs +==== + +.. automodule:: swift.common.middleware.acl + :members: + :show-inheritance: + .. _wsgi: WSGI diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst index a242280553..364a6928dc 100644 --- a/doc/source/overview_auth.rst +++ b/doc/source/overview_auth.rst @@ -6,9 +6,9 @@ The Auth System Developer Auth -------------- -The auth system for Swift is based on the auth system from the existing -Rackspace architecture -- actually from a few existing auth systems -- -and is therefore a bit disjointed. The distilled points about it are: +The auth system for Swift is loosely based on the auth system from the existing +Rackspace architecture -- actually from a few existing auth systems -- and is +therefore a bit disjointed. The distilled points about it are: * The authentication/authorization part is outside Swift itself * The user of Swift passes in an auth token with each request @@ -23,13 +23,16 @@ of something unique, some use "something else" but the salient point is that the token is a string which can be sent as-is back to the auth system for validation. -An auth call is given the auth token and the Swift account hash. For a valid -token, the auth system responds with a session TTL and overall expiration in -seconds from now. Swift does not honor the session TTL but will cache the -token up to the expiration time. Tokens can be purged through a call to the -auth system. +Swift will make calls to the external auth system, giving the auth token to be +validated. For a valid token, the auth system responds with an overall +expiration in seconds from now. Swift will cache the token up to the expiration +time. The included devauth also has the concept of admin and non-admin users +within an account. Admin users can do anything within the account. Non-admin +users can only perform operations per container based on the container's +X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see +:mod:`swift.common.middleware.acl` -The user starts a session by sending a ReST request to that auth system +The user starts a session by sending a ReST request to the external auth system to receive the auth token and a URL to the Swift system. -------------- @@ -40,8 +43,10 @@ Auth is written as wsgi middleware, so implementing your own auth is as easy as writing new wsgi middleware, and plugging it in to the proxy server. The current middleware is implemented in the DevAuthMiddleware class in -swift/common/auth.py, and should be a good starting place for implemeting -your own auth. +swift/common/middleware/auth.py, and should be a good starting place for +implementing your own auth. + +Also, see :doc:`development_auth`. ------------------ History and Future diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 4ae7596392..c3766bfd5d 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -37,6 +37,12 @@ use = egg:swift#proxy [filter:auth] use = egg:swift#auth +# The reseller prefix will verify a token begins with this prefix before even +# attempting to validate it with the external authentication server. Also, with +# authorization, only Swift storage accounts with this prefix will be +# authorized by this middleware. Useful if multiple auth systems are in use for +# one Swift cluster. +# reseller_prefix = AUTH # ip = 127.0.0.1 # port = 11000 # ssl = false diff --git a/setup.py b/setup.py index 2a3084e110..ee8b7fed9d 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ setup( 'bin/st', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/swift-account-replicator', 'bin/swift-account-server', - 'bin/swift-auth-create-account', + 'bin/swift-auth-add-user', 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', 'bin/swift-container-auditor', 'bin/swift-container-replicator', diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 02f1ab76ad..87fa026ebd 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -310,7 +310,7 @@ class AccountReaper(Daemon): try: objects = direct_get_container(node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, - response_timeout=self.node_timeout) + response_timeout=self.node_timeout)[1] self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 except ClientException, err: diff --git a/swift/auth/server.py b/swift/auth/server.py index 5658790c88..38c72114e5 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -22,6 +22,7 @@ from time import gmtime, strftime, time from urllib import unquote, quote from uuid import uuid4 +import sqlite3 from webob import Request, Response from webob.exc import HTTPBadRequest, HTTPNoContent, HTTPUnauthorized, \ HTTPServiceUnavailable, HTTPNotFound @@ -58,17 +59,18 @@ class AuthController(object): * The user makes a ReST call to the Swift cluster using the url given with the token as the X-Auth-Token header. * The Swift cluster makes an ReST call to the auth server to validate the - token for the given account hash, caching the result for future requests - up to the expiration the auth server returns. - * The auth server validates the token / account hash given and returns the - expiration for the token. + token, caching the result for future requests up to the expiration the + auth server returns. + * The auth server validates the token given and returns the expiration for + the token. * The Swift cluster completes the user's request. - Another use case is creating a new account: + Another use case is creating a new user: - * The developer makes a ReST call to create a new account. - * The auth server makes ReST calls to the Swift cluster's account servers - to create a new account on its end. + * The developer makes a ReST call to create a new user. + * If the account for the user does not yet exist, the auth server makes + ReST calls to the Swift cluster's account servers to create a new account + on its end. * The auth server records the information in its database. A last use case is recreating existing accounts; this is really only useful @@ -92,6 +94,9 @@ class AuthController(object): def __init__(self, conf, ring=None): self.logger = get_logger(conf) self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() + if self.reseller_prefix and self.reseller_prefix[-1] != '_': + self.reseller_prefix += '_' self.default_cluster_url = \ conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1') self.token_life = int(conf.get('token_life', 86400)) @@ -103,17 +108,33 @@ class AuthController(object): Ring(os.path.join(self.swift_dir, 'account.ring.gz')) self.db_file = os.path.join(self.swift_dir, 'auth.db') self.conn = get_db_connection(self.db_file, okay_to_create=True) + try: + self.conn.execute('SELECT admin FROM account LIMIT 1') + except sqlite3.OperationalError, err: + if str(err) == 'no such column: admin': + self.conn.execute("ALTER TABLE account ADD COLUMN admin TEXT") + self.conn.execute("UPDATE account SET admin = 't'") self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT)''') + user TEXT, password TEXT, admin TEXT)''') self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account ON account (account)''') + try: + self.conn.execute('SELECT user FROM token LIMIT 1') + except sqlite3.OperationalError, err: + if str(err) == 'no such column: user': + self.conn.execute('DROP INDEX IF EXISTS ix_token_created') + self.conn.execute('DROP INDEX IF EXISTS ix_token_cfaccount') + self.conn.execute('DROP TABLE IF EXISTS token') self.conn.execute('''CREATE TABLE IF NOT EXISTS token ( - cfaccount TEXT, token TEXT, created FLOAT)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount - ON token (cfaccount)''') + token TEXT, created FLOAT, + account TEXT, user TEXT, cfaccount TEXT)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token + ON token (token)''') self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created ON token (created)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account + ON token (account)''') self.conn.commit() def add_storage_account(self, account_name=''): @@ -129,7 +150,7 @@ class AuthController(object): begin = time() orig_account_name = account_name if not account_name: - account_name = str(uuid4()) + account_name = '%s%s' % (self.reseller_prefix, uuid4().hex) partition, nodes = self.account_ring.get_nodes(account_name) headers = {'X-Timestamp': normalize_timestamp(time()), 'x-cf-trans-id': 'tx' + str(uuid4())} @@ -202,77 +223,99 @@ class AuthController(object): (time() - self.token_life,)) conn.commit() - def validate_token(self, token, account_hash): + def validate_token(self, token): """ Tests if the given token is a valid token :param token: The token to validate - :param account_hash: The account hash the token is being used with - :returns: TTL if valid, False otherwise + :returns: (TTL, account, user, cfaccount) if valid, False otherwise. + cfaccount will be None for users without admin access. """ begin = time() self.purge_old_tokens() rv = False with self.get_conn() as conn: row = conn.execute(''' - SELECT created FROM token - WHERE cfaccount = ? AND token = ?''', - (account_hash, token)).fetchone() + SELECT created, account, user, cfaccount FROM token + WHERE token = ?''', + (token,)).fetchone() if row is not None: created = row[0] if time() - created >= self.token_life: conn.execute(''' - DELETE FROM token - WHERE cfaccount = ? AND token = ?''', - (account_hash, token)) + DELETE FROM token WHERE token = ?''', (token,)) conn.commit() else: - rv = self.token_life - (time() - created) - self.logger.info('validate_token(%s, %s, _, _) = %s [%.02f]' % - (repr(token), repr(account_hash), repr(rv), - time() - begin)) + rv = (self.token_life - (time() - created), row[1], row[2], + row[3]) + self.logger.info('validate_token(%s, _, _) = %s [%.02f]' % + (repr(token), repr(rv), time() - begin)) return rv - def create_account(self, new_account, new_user, new_password): + def create_user(self, account, user, password, admin=False): """ - Handles the create_account call for developers, used to request - an account be created both on a Swift cluster and in the auth server - database. + Handles the create_user call for developers, used to request a user be + added in the auth server database. If the account does not yet exist, + it will be created on the Swift cluster and the details recorded in the + auth server database. - This will make ReST requests to the Swift cluster's account servers - to have an account created on its side. The resulting account hash - along with the URL to use to access the account, the account name, the - user name, and the password is recorded in the auth server's database. - The url is constructed now and stored separately to support changing - the configuration file's default_cluster_url for directing new accounts - to a different Swift cluster while still supporting old accounts going - to the Swift clusters they were created on. + The url for the storage account is constructed now and stored + separately to support changing the configuration file's + default_cluster_url for directing new accounts to a different Swift + cluster while still supporting old accounts going to the Swift clusters + they were created on. - :param new_account: The name for the new account - :param new_user: The name for the new user - :param new_password: The password for the new account + Currently, updating a user's information (password, admin access) must + be done by directly updating the sqlite database. - :returns: False if the create fails, storage url if successful + :param account: The name for the new account + :param user: The name for the new user + :param password: The password for the new account + :param admin: If true, the user will be granted full access to the + account; otherwise, another user will have to add the + user to the ACLs for containers to grant access. + + :returns: False if the create fails, 'already exists' if the user + already exists, or storage url if successful """ begin = time() - if not all((new_account, new_user, new_password)): + if not all((account, user, password)): return False - account_hash = self.add_storage_account() - if not account_hash: - self.logger.info( - 'FAILED create_account(%s, %s, _,) [%.02f]' % - (repr(new_account), repr(new_user), time() - begin)) - return False - url = self.default_cluster_url.rstrip('/') + '/' + account_hash with self.get_conn() as conn: + row = conn.execute( + 'SELECT url FROM account WHERE account = ? AND user = ?', + (account, user)).fetchone() + if row: + self.logger.info( + 'ALREADY EXISTS create_user(%s, %s, _, %s) [%.02f]' % + (repr(account), repr(user), repr(admin), + time() - begin)) + return 'already exists' + row = conn.execute( + 'SELECT url, cfaccount FROM account WHERE account = ?', + (account,)).fetchone() + if row: + url = row[0] + account_hash = row[1] + else: + account_hash = self.add_storage_account() + if not account_hash: + self.logger.info( + 'FAILED create_user(%s, %s, _, %s) [%.02f]' % + (repr(account), repr(user), repr(admin), + time() - begin)) + return False + url = self.default_cluster_url.rstrip('/') + '/' + account_hash conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password) - VALUES (?, ?, ?, ?, ?)''', - (new_account, url, account_hash, new_user, new_password)) + (account, url, cfaccount, user, password, admin) + VALUES (?, ?, ?, ?, ?, ?)''', + (account, url, account_hash, user, password, + admin and 't' or '')) conn.commit() self.logger.info( - 'SUCCESS create_account(%s, %s, _) = %s [%.02f]' % - (repr(new_account), repr(new_user), repr(url), time() - begin)) + 'SUCCESS create_user(%s, %s, _, %s) = %s [%.02f]' % + (repr(account), repr(user), repr(admin), repr(url), + time() - begin)) return url def recreate_accounts(self): @@ -285,8 +328,8 @@ class AuthController(object): """ begin = time() with self.get_conn() as conn: - account_hashes = [r[0] for r in - conn.execute('SELECT cfaccount FROM account').fetchall()] + account_hashes = [r[0] for r in conn.execute( + 'SELECT distinct(cfaccount) FROM account').fetchall()] failures = [] for i, account_hash in enumerate(account_hashes): if not self.add_storage_account(account_hash): @@ -298,36 +341,47 @@ class AuthController(object): def handle_token(self, request): """ - Hanles ReST request from Swift to validate tokens + Handles ReST requests from Swift to validate tokens Valid URL paths: - * GET /token// + * GET /token/ - If the HTTP equest returns with a 204, then the token is valid, - and the TTL of the token will be available in the X-Auth-Ttl header. + If the HTTP request returns with a 204, then the token is valid, the + TTL of the token will be available in the X-Auth-Ttl header, and a + comma separated list of the "groups" the user belongs to will be in the + X-Auth-Groups header. :param request: webob.Request object """ try: - _, account_hash, token = split_path(request.path, minsegs=3) + _, token = split_path(request.path, minsegs=2) except ValueError: return HTTPBadRequest() - ttl = self.validate_token(token, account_hash) - if not ttl: + # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise + validation = self.validate_token(token) + if not validation: return HTTPNotFound() - return HTTPNoContent(headers={'x-auth-ttl': ttl}) + groups = ['%s:%s' % (validation[1], validation[2]), validation[1]] + if validation[3]: # admin access to a cfaccount + groups.append(validation[3]) + return HTTPNoContent(headers={'X-Auth-TTL': validation[0], + 'X-Auth-Groups': ','.join(groups)}) - def handle_account_create(self, request): + def handle_add_user(self, request): """ - Handles Rest requests from developers to have an account created. + Handles Rest requests from developers to have a user added. If the + account specified doesn't exist, it will also be added. Currently, + updating a user's information (password, admin access) must be done by + directly updating the sqlite database. Valid URL paths: * PUT /account// - create the account Valid headers: - * X-Auth-Key: (Only required when creating an account) + * X-Auth-User-Key: + * X-Auth-User-Admin: - If the HTTP request returns with a 204, then the account was created, + If the HTTP request returns with a 204, then the user was added, and the storage url will be available in the X-Storage-Url header. :param request: webob.Request object @@ -336,10 +390,13 @@ class AuthController(object): _, account_name, user_name = split_path(request.path, minsegs=3) except ValueError: return HTTPBadRequest() - if 'X-Auth-Key' not in request.headers: - return HTTPBadRequest('X-Auth-Key is required') - password = request.headers['x-auth-key'] - storage_url = self.create_account(account_name, user_name, password) + if 'X-Auth-User-Key' not in request.headers: + return HTTPBadRequest('X-Auth-User-Key is required') + password = request.headers['x-auth-user-key'] + storage_url = self.create_user(account_name, user_name, password, + request.headers.get('x-auth-user-admin') == 'true') + if storage_url == 'already exists': + return HTTPBadRequest(storage_url) if not storage_url: return HTTPServiceUnavailable() return HTTPNoContent(headers={'x-storage-url': storage_url}) @@ -414,23 +471,26 @@ class AuthController(object): self.purge_old_tokens() with self.get_conn() as conn: row = conn.execute(''' - SELECT cfaccount, url FROM account + SELECT cfaccount, url, admin FROM account WHERE account = ? AND user = ? AND password = ?''', (account, user, password)).fetchone() if row is None: return HTTPUnauthorized() cfaccount = row[0] url = row[1] - row = conn.execute('SELECT token FROM token WHERE cfaccount = ?', - (cfaccount,)).fetchone() + admin = row[2] == 't' + row = conn.execute(''' + SELECT token FROM token WHERE account = ? AND user = ?''', + (account, user)).fetchone() if row: token = row[0] else: - token = 'tk' + str(uuid4()) + token = '%stk%s' % (self.reseller_prefix, uuid4().hex) conn.execute(''' - INSERT INTO token (cfaccount, token, created) - VALUES (?, ?, ?)''', - (cfaccount, token, time())) + INSERT INTO token + (token, created, account, user, cfaccount) + VALUES (?, ?, ?, ?, ?)''', + (token, time(), account, user, admin and cfaccount or '')) conn.commit() return HTTPNoContent(headers={'x-auth-token': token, 'x-storage-token': token, @@ -457,7 +517,7 @@ class AuthController(object): elif req.method == 'GET' and req.path.startswith('/token/'): handler = self.handle_token elif req.method == 'PUT' and req.path.startswith('/account/'): - handler = self.handle_account_create + handler = self.handle_add_user elif req.method == 'POST' and \ req.path == '/recreate_accounts': handler = self.handle_account_recreate diff --git a/swift/common/client.py b/swift/common/client.py index c9ff71e274..e17afee958 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -194,18 +194,21 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of accounts + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_account(url, token, marker, limit, prefix, http_conn) + rv = get_account(url, token, marker, limit, prefix, http_conn) + listing = rv[1] while listing: - rv.extend(listing) marker = listing[-1]['name'] - listing = get_account(url, token, marker, limit, prefix, http_conn) + listing = \ + get_account(url, token, marker, limit, prefix, http_conn)[1] + if listing: + rv.extend(listing) return rv parsed, conn = http_conn qs = 'format=json' @@ -218,6 +221,9 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn.request('GET', '%s?%s' % (parsed.path, qs), '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException('Account GET failed', http_scheme=parsed.scheme, @@ -226,8 +232,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, http_reason=resp.reason) if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_account(url, token, http_conn=None): @@ -238,7 +244,8 @@ def head_account(url, token, http_conn=None): :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (container count, object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -253,9 +260,36 @@ def head_account(url, token, http_conn=None): http_host=conn.host, http_port=conn.port, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-account-container-count', 0)), \ - int(resp.getheader('x-account-object-count', 0)), \ - int(resp.getheader('x-account-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + +def post_account(url, token, headers, http_conn=None): + """ + Update an account's metadata. + + :param url: storage URL + :param token: auth token + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + headers['X-Auth-Token'] = token + conn.request('POST', parsed.path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) def get_container(url, token, container, marker=None, limit=None, @@ -275,23 +309,25 @@ def get_container(url, token, container, marker=None, limit=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) + rv = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + listing = rv[1] while listing: - rv.extend(listing) if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn) + prefix, delimiter, http_conn)[1] + if listing: + rv[1].extend(listing) return rv parsed, conn = http_conn path = '%s/%s' % (parsed.path, quote(container)) @@ -312,10 +348,13 @@ def get_container(url, token, container, marker=None, limit=None, http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_query=qs, http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_container(url, token, container, http_conn=None): @@ -327,7 +366,8 @@ def head_container(url, token, container, http_conn=None): :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -343,17 +383,20 @@ def head_container(url, token, container, http_conn=None): http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count', 0)), \ - int(resp.getheader('x-container-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers -def put_container(url, token, container, http_conn=None): +def put_container(url, token, container, headers=None, http_conn=None): """ Create a container :param url: storage URL :param token: auth token :param container: container name to create + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP PUT request failed @@ -363,7 +406,10 @@ def put_container(url, token, container, http_conn=None): else: parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) - conn.request('PUT', path, '', {'X-Auth-Token': token}) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + conn.request('PUT', path, '', headers) resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: @@ -373,6 +419,34 @@ def put_container(url, token, container, http_conn=None): http_reason=resp.reason) +def post_container(url, token, container, headers, http_conn=None): + """ + Update a container's metadata. + + :param url: storage URL + :param token: auth token + :param container: container name to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + def delete_container(url, token, container, http_conn=None): """ Delete a container @@ -410,8 +484,12 @@ def get_object(url, token, container, name, http_conn=None, :param name: object name to get :param http_conn: HTTP connection object (If None, it will create the conn object) - :param resp_chunk_size: if defined, chunk size of data to read - :returns: a list of objects + :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If + you specify a resp_chunk_size you must fully read + the object's contents before making another + request. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if http_conn: @@ -426,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None, raise ClientException('Object GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) if resp_chunk_size: def _object_body(): @@ -440,12 +514,10 @@ def get_object(url, token, container, name, http_conn=None, object_body = _object_body() else: object_body = resp.read() - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata, \ - object_body + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def head_object(url, token, container, name, http_conn=None): @@ -458,8 +530,8 @@ def head_object(url, token, container, name, http_conn=None): :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (content type, content length, last modfied, etag, - dictionary of metadata) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -474,20 +546,15 @@ def head_object(url, token, container, name, http_conn=None): raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers -def put_object(url, token, container, name, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None, http_conn=None): +def put_object(url, token, container, name, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, headers=None, + http_conn=None): """ Put an object @@ -495,12 +562,12 @@ def put_object(url, token, container, name, contents, metadata={}, :param token: auth token :param container: container name that the object is in :param name: object name to put - :param contents: file like object to read object data from - :param metadata: dictionary of object metadata + :param contents: a string or a file like object to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param chunk_size: chunk size of data to write :param content_type: value to send as content-type header + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :returns: etag from server response @@ -511,9 +578,9 @@ def put_object(url, token, container, name, contents, metadata={}, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if not headers: + headers = {} + headers['X-Auth-Token'] = token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -549,15 +616,15 @@ def put_object(url, token, container, name, contents, metadata={}, return resp.getheader('etag').strip('"') -def post_object(url, token, container, name, metadata, http_conn=None): +def post_object(url, token, container, name, headers, http_conn=None): """ - Change object metadata + Update object metadata :param url: storage URL :param token: auth token :param container: container name that the object is in - :param name: object name to change - :param metadata: dictionary of object metadata + :param name: name of the object to update + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP POST request failed @@ -567,9 +634,7 @@ def post_object(url, token, container, name, metadata, http_conn=None): else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + headers['X-Auth-Token'] = token conn.request('POST', path, '', headers) resp = conn.getresponse() resp.read() @@ -682,6 +747,10 @@ class Connection(object): return self._retry(get_account, marker=marker, limit=limit, prefix=prefix, full_listing=full_listing) + def post_account(self, headers): + """Wrapper for :func:`post_account`""" + return self._retry(post_account, headers) + def head_container(self, container): """Wrapper for :func:`head_container`""" return self._retry(head_container, container) @@ -696,9 +765,13 @@ class Connection(object): limit=limit, prefix=prefix, delimiter=delimiter, full_listing=full_listing) - def put_container(self, container): + def put_container(self, container, headers=None): """Wrapper for :func:`put_container`""" - return self._retry(put_container, container) + return self._retry(put_container, container, headers=headers) + + def post_container(self, container, headers): + """Wrapper for :func:`post_container`""" + return self._retry(post_container, container, headers) def delete_container(self, container): """Wrapper for :func:`delete_container`""" @@ -713,17 +786,17 @@ class Connection(object): return self._retry(get_object, container, obj, resp_chunk_size=resp_chunk_size) - def put_object(self, container, obj, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None): + def put_object(self, container, obj, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, + headers=None): """Wrapper for :func:`put_object`""" return self._retry(put_object, container, obj, contents, - metadata=metadata, content_length=content_length, etag=etag, - chunk_size=chunk_size, content_type=content_type) + content_length=content_length, etag=etag, chunk_size=chunk_size, + content_type=content_type, headers=headers) - def post_object(self, container, obj, metadata): + def post_object(self, container, obj, headers): """Wrapper for :func:`post_object`""" - return self._retry(post_object, container, obj, metadata) + return self._retry(post_object, container, obj, headers) def delete_object(self, container, obj): """Wrapper for :func:`delete_object`""" diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 1d7030c09d..71fe3005a4 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5, :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s' % (account, container) with Timeout(conn_timeout): @@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count')), \ - int(resp.getheader('x-container-bytes-used')) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_container(node, part, account, container, marker=None, @@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None, :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s' % (account, container) qs = 'format=json' @@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def direct_delete_container(node, part, account, container, conn_timeout=5, @@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Container server %s:%s direct DELETE %s gave status %s' % @@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def direct_head_object(node, part, account, container, obj, conn_timeout=5, @@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (content-type, object size, last modified timestamp, - etag, metadata dictionary) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length')), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_object(node, part, account, container, obj, conn_timeout=5, - response_timeout=15): + response_timeout=15, resp_chunk_size=None): """ Get object directly from the object server. @@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: object + :param resp_chunk_size: if defined, chunk size of data to read. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, with Timeout(response_timeout): resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: + resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], @@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return (resp.getheader('content-type'), - int(resp.getheader('content-length')), - resp.getheader('last-modified'), - resp.getheader('etag').strip('"'), - metadata, - resp.read()) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def direct_delete_object(node, part, account, container, obj, @@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Object server %s:%s direct DELETE %s gave status %s' % @@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def retry(func, *args, **kwargs): diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py new file mode 100644 index 0000000000..6403aed726 --- /dev/null +++ b/swift/common/middleware/acl.py @@ -0,0 +1,160 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from urlparse import urlparse + + +def clean_acl(name, value): + """ + Returns a cleaned ACL header value, validating that it meets the formatting + requirements for standard Swift ACL strings. + + The ACL format is:: + + [item[,item...]] + + Each item can be a group name to give access to or a referrer designation + to grant or deny based on the HTTP Referer header. + + The referrer designation format is:: + + .r:[-]value + + The ``.r`` can also be ``.ref``, ``.referer``, or ``.referrer``; though it + will be shortened to just ``.r`` for decreased character count usage. + + The value can be ``*`` to specify any referrer host is allowed access, a + specific host name like ``www.example.com``, or if it has a leading period + ``.`` or leading ``*.`` it is a domain name specification, like + ``.example.com`` or ``*.example.com``. The leading minus sign ``-`` + indicates referrer hosts that should be denied access. + + Referrer access is applied in the order they are specified. For example, + .r:.example.com,.r:-thief.example.com would allow all hosts ending with + .example.com except for the specific host thief.example.com. + + Example valid ACLs:: + + .r:* + .r:*,.r:-.thief.com + .r:*,.r:.example.com,.r:-thief.example.com + .r:*,.r:-.thief.com,bobs_account,sues_account:sue + bobs_account,sues_account:sue + + Example invalid ACLs:: + + .r: + .r:- + + Also, .r designations aren't allowed in headers whose names include the + word 'write'. + + ACLs that are "messy" will be cleaned up. Examples: + + ====================== ====================== + Original Cleaned + ---------------------- ---------------------- + ``bob, sue`` ``bob,sue`` + ``bob , sue`` ``bob,sue`` + ``bob,,,sue`` ``bob,sue`` + ``.referrer : *`` ``.r:*`` + ``.ref:*.example.com`` ``.r:.example.com`` + ====================== ====================== + + :param name: The name of the header being cleaned, such as X-Container-Read + or X-Container-Write. + :param value: The value of the header being cleaned. + :returns: The value, cleaned of extraneous formatting. + :raises ValueError: If the value does not meet the ACL formatting + requirements; the error message will indicate why. + """ + name = name.lower() + values = [] + for raw_value in value.split(','): + raw_value = raw_value.strip() + if raw_value: + if ':' not in raw_value: + values.append(raw_value) + else: + first, second = (v.strip() for v in raw_value.split(':', 1)) + if not first or first[0] != '.': + values.append(raw_value) + elif first in ('.r', '.ref', '.referer', '.referrer'): + if 'write' in name: + raise ValueError('Referrers not allowed in write ACL: ' + '%s' % repr(raw_value)) + negate = False + if second and second[0] == '-': + negate = True + second = second[1:].strip() + if second and second != '*' and second[0] == '*': + second = second[1:].strip() + if not second or second == '.': + raise ValueError('No host/domain value after referrer ' + 'designation in ACL: %s' % repr(raw_value)) + values.append('.r:%s%s' % (negate and '-' or '', second)) + else: + raise ValueError('Unknown designator %s in ACL: %s' % + (repr(first), repr(raw_value))) + return ','.join(values) + + +def parse_acl(acl_string): + """ + Parses a standard Swift ACL string into a referrers list and groups list. + + See :func:`clean_acl` for documentation of the standard Swift ACL format. + + :param acl_string: The standard Swift ACL string to parse. + :returns: A tuple of (referrers, groups) where referrers is a list of + referrer designations (without the leading .r:) and groups is a + list of groups to allow access. + """ + referrers = [] + groups = [] + if acl_string: + for value in acl_string.split(','): + if value.startswith('.r:'): + referrers.append(value[len('.r:'):]) + else: + groups.append(value) + return referrers, groups + + +def referrer_allowed(referrer, referrer_acl): + """ + Returns True if the referrer should be allowed based on the referrer_acl + list (as returned by :func:`parse_acl`). + + See :func:`clean_acl` for documentation of the standard Swift ACL format. + + :param referrer: The value of the HTTP Referer header. + :param referrer_acl: The list of referrer designations as returned by + :func:`parse_acl`. + :returns: True if the referrer should be allowed; False if not. + """ + allow = False + if referrer_acl: + rhost = urlparse(referrer or '').hostname or 'unknown' + for mhost in referrer_acl: + if mhost[0] == '-': + mhost = mhost[1:] + if mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = False + elif mhost == '*' or mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = True + return allow diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index eb920fad41..03770175fd 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -13,31 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time +from time import time -from webob.request import Request -from webob.exc import HTTPUnauthorized, HTTPPreconditionFailed from eventlet.timeout import Timeout +from webob.exc import HTTPForbidden, HTTPUnauthorized -from swift.common.utils import split_path from swift.common.bufferedhttp import http_connect_raw as http_connect -from swift.common.utils import get_logger, cache_from_env -from swift.common.memcached import MemcacheRing +from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed +from swift.common.utils import cache_from_env, split_path -class DevAuthMiddleware(object): - """ - Auth Middleware that uses the dev auth server - """ +class DevAuth(object): + """Auth Middleware that uses the dev auth server.""" - def __init__(self, app, conf, memcache_client=None, logger=None): + def __init__(self, app, conf): self.app = app - self.memcache_client = memcache_client - if logger is None: - self.logger = get_logger(conf) - else: - self.logger = logger self.conf = conf + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() + if self.reseller_prefix and self.reseller_prefix[-1] != '_': + self.reseller_prefix += '_' self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) self.ssl = \ @@ -45,68 +39,79 @@ class DevAuthMiddleware(object): self.timeout = int(conf.get('node_timeout', 10)) def __call__(self, env, start_response): - if self.memcache_client is None: - self.memcache_client = cache_from_env(env) - req = Request(env) - if 'x-storage-token' in req.headers and \ - 'x-auth-token' not in req.headers: - req.headers['x-auth-token'] = req.headers['x-storage-token'] - try: - version, account, container, obj = split_path(req.path, 1, 4, True) - except ValueError, e: - version = account = container = obj = None - if account is None: - return HTTPPreconditionFailed(request=req, body='Bad URL')( - env, start_response) - if not req.headers.get('x-auth-token'): - return HTTPPreconditionFailed(request=req, - body='Missing Auth Token')(env, start_response) - if not self.auth(account, req.headers['x-auth-token']): - return HTTPUnauthorized(request=req)(env, start_response) - - # If we get here, then things should be good. + """ + Accepts a standard WSGI application call, authenticating the request + and installing callback hooks for authorization and ACL header + validation. For an authenticated request, REMOTE_USER will be set to a + comma separated list of the user's groups. + """ + groups = None + token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) + if token and token.startswith(self.reseller_prefix): + memcache_client = cache_from_env(env) + key = 'devauth/%s' % token + cached_auth_data = memcache_client.get(key) + if cached_auth_data: + start, expiration, groups = cached_auth_data + if time() - start > expiration: + groups = None + if not groups: + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s' % token, ssl=self.ssl) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status // 100 != 2: + return HTTPUnauthorized()(env, start_response) + expiration = float(resp.getheader('x-auth-ttl')) + groups = resp.getheader('x-auth-groups') + memcache_client.set(key, (time(), expiration, groups), + timeout=expiration) + env['REMOTE_USER'] = groups + env['swift.authorize'] = self.authorize + env['swift.clean_acl'] = clean_acl + # We know the proxy logs the token, so we augment it just a bit to also + # log the authenticated user. + user = groups and groups.split(',', 1)[0] or '' + env['HTTP_X_AUTH_TOKEN'] = '%s,%s' % (user, token) return self.app(env, start_response) - def auth(self, account, token): + def authorize(self, req): """ - Dev authorization implmentation - - :param account: account name - :param token: auth token - - :returns: True if authorization is successful, False otherwise + Returns None if the request is authorized to continue or a standard + WSGI response callable if not. """ - key = 'auth/%s/%s' % (account, token) - now = time.time() - cached_auth_data = self.memcache_client.get(key) - if cached_auth_data: - start, expiration = cached_auth_data - if now - start <= expiration: - return True - try: - with Timeout(self.timeout): - conn = http_connect(self.auth_host, self.auth_port, 'GET', - '/token/%s/%s' % (account, token), ssl=self.ssl) - resp = conn.getresponse() - resp.read() - conn.close() - if resp.status == 204: - validated = float(resp.getheader('x-auth-ttl')) - else: - validated = False - except: - self.logger.exception('ERROR with auth') - return False - if not validated: - return False + version, account, container, obj = split_path(req.path, 1, 4, True) + if not account or not account.startswith(self.reseller_prefix): + return self.denied_response(req) + if req.remote_user and account in req.remote_user.split(','): + return None + referrers, groups = parse_acl(getattr(req, 'acl', None)) + if referrer_allowed(req.referer, referrers): + return None + if not req.remote_user: + return self.denied_response(req) + for user_group in req.remote_user.split(','): + if user_group in groups: + return None + return self.denied_response(req) + + def denied_response(self, req): + """ + Returns a standard WSGI response callable with the status of 403 or 401 + depending on whether the REMOTE_USER is set or not. + """ + if req.remote_user: + return HTTPForbidden(request=req) else: - val = (now, validated) - self.memcache_client.set(key, val, timeout=validated) - return True + return HTTPUnauthorized(request=req) + def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) def auth_filter(app): - return DevAuthMiddleware(app, conf) + return DevAuth(app, conf) return auth_filter diff --git a/swift/container/server.py b/swift/container/server.py index 23a9d0b71c..63a44fcabc 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -44,6 +44,9 @@ DATADIR = 'containers' class ContainerController(object): """WSGI Controller for the container server.""" + # Ensure these are all lowercase + save_headers = ['x-container-read', 'x-container-write'] + def __init__(self, conf): self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node/') @@ -192,7 +195,8 @@ class ContainerController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-container-meta-')) + if key.lower() in self.save_headers or + key.lower().startswith('x-container-meta-')) if metadata: broker.update_metadata(metadata) resp = self.account_update(req, account, container, broker) @@ -373,7 +377,8 @@ class ContainerController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-container-meta-')) + if key.lower() in self.save_headers or + key.lower().startswith('x-container-meta-')) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 140bc5dbc4..892c4f4769 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -17,6 +17,7 @@ from __future__ import with_statement import mimetypes import os import time +import traceback from ConfigParser import ConfigParser from urllib import unquote, quote import uuid @@ -73,6 +74,22 @@ def public(func): return wrapped +def delay_denial(func): + """ + Decorator to declare which methods should have any swift.authorize call + delayed. This is so the method can load the Request object up with + additional information that may be needed by the authorization system. + + :param func: function to delay authorization on + """ + func.delay_denial = True + + @functools.wraps(func) + def wrapped(*a, **kw): + return func(*a, **kw) + return wrapped + + class Controller(object): """Base WSGI controller class for the proxy""" @@ -206,19 +223,28 @@ class Controller(object): :param account: account name for the container :param container: container name to look up - :returns: tuple of (container partition, container nodes) or - (None, None) if the container does not exist + :returns: tuple of (container partition, container nodes, container + read acl, container write acl) or (None, None, None, None) if + the container does not exist """ partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) cache_key = 'container%s' % path + # Older memcache values (should be treated as if they aren't there): # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses - if self.app.memcache.get(cache_key) == 200: - return partition, nodes + # Newer memcache values: + # [older status value from above, read acl, write acl] + cache_value = self.app.memcache.get(cache_key) + if hasattr(cache_value, '__iter__'): + status, read_acl, write_acl = cache_value + if status == 200: + return partition, nodes, read_acl, write_acl if not self.account_info(account)[1]: - return (None, None) + return (None, None, None, None) result_code = 0 + read_acl = None + write_acl = None attempts_left = self.app.container_ring.replica_count headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.container_ring): @@ -233,6 +259,8 @@ class Controller(object): body = resp.read() if 200 <= resp.status <= 299: result_code = 200 + read_acl = resp.getheader('x-container-read') + write_acl = resp.getheader('x-container-write') break elif resp.status == 404: result_code = 404 if not result_code else -1 @@ -251,10 +279,11 @@ class Controller(object): cache_timeout = self.app.recheck_container_existence else: cache_timeout = self.app.recheck_container_existence * 0.1 - self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) + self.app.memcache.set(cache_key, (result_code, read_acl, write_acl), + timeout=cache_timeout) if result_code == 200: - return partition, nodes - return (None, None) + return partition, nodes, read_acl, write_acl + return (None, None, None, None) def iter_nodes(self, partition, nodes, ring): """ @@ -474,6 +503,12 @@ class ObjectController(Controller): def GETorHEAD(self, req): """Handle HTTP GET or HEAD requests.""" + if 'swift.authorize' in req.environ: + req.acl = \ + self.container_info(self.account_name, self.container_name)[2] + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) return self.GETorHEAD_base(req, 'Object', partition, @@ -481,13 +516,30 @@ class ObjectController(Controller): req.path_info, self.app.object_ring.replica_count) @public + @delay_denial + def GET(self, req): + """Handler for HTTP GET requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial + def HEAD(self, req): + """Handler for HTTP HEAD requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial def POST(self, req): """HTTP POST request handler.""" error_response = check_metadata(req, 'object') if error_response: return error_response - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -521,10 +573,15 @@ class ObjectController(Controller): bodies, 'Object POST') @public + @delay_denial def PUT(self, req): """HTTP PUT request handler.""" - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -701,10 +758,15 @@ class ObjectController(Controller): return resp @public + @delay_denial def DELETE(self, req): """HTTP DELETE request handler.""" - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -771,11 +833,26 @@ class ObjectController(Controller): class ContainerController(Controller): """WSGI controller for container requests""" + # Ensure these are all lowercase + pass_through_headers = ['x-container-read', 'x-container-write'] + def __init__(self, app, account_name, container_name, **kwargs): Controller.__init__(self, app) self.account_name = unquote(account_name) self.container_name = unquote(container_name) + def clean_acls(self, req): + if 'swift.clean_acl' in req.environ: + for header in ('x-container-read', 'x-container-write'): + if header in req.headers: + try: + req.headers[header] = \ + req.environ['swift.clean_acl'](header, + req.headers[header]) + except ValueError, err: + return HTTPBadRequest(request=req, body=str(err)) + return None + def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if not self.account_info(self.account_name)[1]: @@ -784,12 +861,30 @@ class ContainerController(Controller): self.account_name, self.container_name) resp = self.GETorHEAD_base(req, 'Container', part, nodes, req.path_info, self.app.container_ring.replica_count) + if 'swift.authorize' in req.environ: + req.acl = resp.headers.get('x-container-read') + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp return resp + @public + @delay_denial + def GET(self, req): + """Handler for HTTP GET requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial + def HEAD(self, req): + """Handler for HTTP HEAD requests.""" + return self.GETorHEAD(req) + @public def PUT(self, req): """HTTP PUT request handler.""" - error_response = check_metadata(req, 'container') + error_response = \ + self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH: @@ -807,7 +902,8 @@ class ContainerController(Controller): headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() - if value[0].lower().startswith('x-container-meta-')) + if value[0].lower() in self.pass_through_headers or + value[0].lower().startswith('x-container-meta-')) statuses = [] reasons = [] bodies = [] @@ -853,7 +949,8 @@ class ContainerController(Controller): @public def POST(self, req): """HTTP POST request handler.""" - error_response = check_metadata(req, 'container') + error_response = \ + self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response account_partition, accounts = self.account_info(self.account_name) @@ -864,7 +961,8 @@ class ContainerController(Controller): headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} headers.update(value for value in req.headers.iteritems() - if value[0].lower().startswith('x-container-meta-')) + if value[0].lower() in self.pass_through_headers or + value[0].lower().startswith('x-container-meta-')) statuses = [] reasons = [] bodies = [] @@ -1118,7 +1216,8 @@ class BaseApplication(object): self.posthooklogger(env, req) return response except: - print "EXCEPTION IN __call__: %s" % env + print "EXCEPTION IN __call__: %s: %s" % \ + (traceback.format_exc(), env) start_response('500 Server Error', [('Content-Type', 'text/plain')]) return ['Internal server error.\n'] @@ -1160,12 +1259,30 @@ class BaseApplication(object): controller.trans_id = req.headers.get('x-cf-trans-id', '-') try: handler = getattr(controller, req.method) - if getattr(handler, 'publicly_accessible'): - if path_parts['version']: - req.path_info_pop() - return handler(req) + if not getattr(handler, 'publicly_accessible'): + handler = None except AttributeError: + handler = None + if not handler: return HTTPMethodNotAllowed(request=req) + if path_parts['version']: + req.path_info_pop() + if 'swift.authorize' in req.environ: + # We call authorize before the handler, always. If authorized, + # we remove the swift.authorize hook so isn't ever called + # again. If not authorized, we return the denial unless the + # controller's method indicates it'd like to gather more + # information and try again later. + resp = req.environ['swift.authorize'](req) + if not resp: + # No resp means authorized, no delayed recheck required. + del req.environ['swift.authorize'] + else: + # Response indicates denial, but we might delay the denial + # and recheck later. If not delayed, return the error now. + if not getattr(handler, 'delay_denial', None): + return resp + return handler(req) except Exception: self.logger.exception('ERROR Unhandled exception in request') return HTTPServerError(request=req) @@ -1187,7 +1304,9 @@ class Application(BaseApplication): return req.response def posthooklogger(self, env, req): - response = req.response + response = getattr(req, 'response', None) + if not response: + return trans_time = '%.4f' % (time.time() - req.start_time) the_request = quote(unquote(req.path)) if req.query_string: diff --git a/test/functional/sample.conf b/test/functional/sample.conf index dbcd969e74..983f2cf768 100644 --- a/test/functional/sample.conf +++ b/test/functional/sample.conf @@ -1,10 +1,20 @@ -# Sample functional test configuration file +# sample config auth_host = 127.0.0.1 -auth_port = 80 +auth_port = 11000 auth_ssl = no -account = test_account -username = test_user -password = test_password +# Primary functional test account (needs admin access to the account) +account = test +username = tester +password = testing + +# User on a second account (needs admin access to the account) +account2 = test2 +username2 = tester2 +password2 = testing2 + +# User on same account as first, but without admin access +username3 = tester3 +password3 = testing3 collate = C diff --git a/test/functional/tests.py b/test/functional/tests.py index 3a1f35fc36..ae396f3fb0 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -106,9 +106,12 @@ class Base(unittest.TestCase): self.assert_(response_body == body, 'Body returned: %s' % (response_body)) - def assert_status(self, status): - self.assert_(self.env.conn.response.status == status, - 'Status returned: %d' % (self.env.conn.response.status)) + def assert_status(self, status_or_statuses): + self.assert_(self.env.conn.response.status == status_or_statuses or + (hasattr(status_or_statuses, '__iter__') and + self.env.conn.response.status in status_or_statuses), + 'Status returned: %d Expected: %s' % + (self.env.conn.response.status, status_or_statuses)) class Base2(object): def setUp(self): @@ -148,11 +151,11 @@ class TestAccount(Base): def testNoAuthToken(self): self.assertRaises(ResponseError, self.env.account.info, cfg={'no_auth_token':True}) - self.assert_status(412) + self.assert_status([401, 412]) self.assertRaises(ResponseError, self.env.account.containers, cfg={'no_auth_token':True}) - self.assert_status(412) + self.assert_status([401, 412]) def testInvalidUTF8Path(self): invalid_utf8 = Utils.create_utf8_name()[::-1] @@ -1123,7 +1126,8 @@ class TestFile(Base): self.assert_status(400) # bad request types - for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'): + #for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'): + for req in ('LICK', 'GETorHEAD_base'): self.env.account.conn.make_request(req) self.assert_status(405) diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py index 1c805f854d..8bd46b462b 100644 --- a/test/functionalnosetests/swift_testing.py +++ b/test/functionalnosetests/swift_testing.py @@ -10,11 +10,11 @@ from swift.common.client import get_auth, http_connection swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') -swift_test_user = os.environ.get('SWIFT_TEST_USER') -swift_test_key = os.environ.get('SWIFT_TEST_KEY') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] # If no environment set, fall back to old school conf file -if not all([swift_test_auth, swift_test_user, swift_test_key]): +if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]): conf = ConfigParser() class Sectionizer(object): def __init__(self, fp): @@ -32,16 +32,36 @@ if not all([swift_test_auth, swift_test_user, swift_test_key]): if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): swift_test_auth = 'https' swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf - swift_test_user = '%(account)s:%(username)s' % conf - swift_test_key = conf['password'] + swift_test_user[0] = '%(account)s:%(username)s' % conf + swift_test_key[0] = conf['password'] + try: + swift_test_user[1] = '%(account2)s:%(username2)s' % conf + swift_test_key[1] = conf['password2'] + except KeyError, err: + pass # old conf, no second account tests can be run + try: + swift_test_user[2] = '%(account)s:%(username3)s' % conf + swift_test_key[2] = conf['password3'] + except KeyError, err: + pass # old conf, no third account tests can be run except IOError, err: if err.errno != errno.ENOENT: raise -skip = not all([swift_test_auth, swift_test_user, swift_test_key]) +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) if skip: print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) +if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) +if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + class AuthError(Exception): pass @@ -51,29 +71,44 @@ class InternalServerError(Exception): pass -url = token = parsed = conn = None +url = [None, None, None] +token = [None, None, None] +parsed = [None, None, None] +conn = [None, None, None] def retry(func, *args, **kwargs): + """ + You can use the kwargs to override the 'retries' (default: 5) and + 'use_account' (default: 1). + """ global url, token, parsed, conn retries = kwargs.get('retries', 5) + use_account = 1 + if 'use_account' in kwargs: + use_account = kwargs['use_account'] + del kwargs['use_account'] + use_account -= 1 attempts = 0 backoff = 1 while attempts <= retries: attempts += 1 try: - if not url or not token: - url, token = \ - get_auth(swift_test_auth, swift_test_user, swift_test_key) - parsed = conn = None - if not parsed or not conn: - parsed, conn = http_connection(url) - return func(url, token, parsed, conn, *args, **kwargs) + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = \ + get_auth(swift_test_auth, swift_test_user[use_account], + swift_test_key[use_account]) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + http_connection(url[use_account]) + return func(url[use_account], token[use_account], + parsed[use_account], conn[use_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: raise - parsed = conn = None + parsed[use_account] = conn[use_account] = None except AuthError, err: - url = token = None + url[use_account] = token[use_account] = None continue except InternalServerError, err: pass diff --git a/test/functionalnosetests/test_account.py b/test/functionalnosetests/test_account.py index dea19380ac..4b5da32da1 100755 --- a/test/functionalnosetests/test_account.py +++ b/test/functionalnosetests/test_account.py @@ -1,6 +1,7 @@ #!/usr/bin/python import unittest +from nose import SkipTest from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH @@ -12,7 +13,7 @@ class TestAccount(unittest.TestCase): def test_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, value): conn.request('POST', parsed.path, '', {'X-Auth-Token': token, 'X-Account-Meta-Test': value}) @@ -48,7 +49,7 @@ class TestAccount(unittest.TestCase): def test_multi_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path, '', {'X-Auth-Token': token, name: value}) @@ -74,7 +75,7 @@ class TestAccount(unittest.TestCase): def test_bad_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py index 9c36b460b8..96c0be91e6 100755 --- a/test/functionalnosetests/test_container.py +++ b/test/functionalnosetests/test_container.py @@ -1,19 +1,22 @@ #!/usr/bin/python +import json import unittest +from nose import SkipTest from uuid import uuid4 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH -from swift_testing import check_response, retry, skip +from swift_testing import check_response, retry, skip, skip2, skip3, \ + swift_test_user class TestContainer(unittest.TestCase): def setUp(self): if skip: - return + raise SkipTest self.name = uuid4().hex def put(url, token, parsed, conn): conn.request('PUT', parsed.path + '/' + self.name, '', @@ -25,7 +28,27 @@ class TestContainer(unittest.TestCase): def tearDown(self): if skip: - return + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + while True: + resp = retry(get) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) @@ -36,7 +59,7 @@ class TestContainer(unittest.TestCase): def test_multi_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, name, value): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, name: value}) @@ -63,7 +86,7 @@ class TestContainer(unittest.TestCase): def test_PUT_metadata(self): if skip: - return + raise SkipTest def put(url, token, parsed, conn, name, value): conn.request('PUT', parsed.path + '/' + name, '', {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) @@ -110,7 +133,7 @@ class TestContainer(unittest.TestCase): def test_POST_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, value): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, 'X-Container-Meta-Test': value}) @@ -145,7 +168,7 @@ class TestContainer(unittest.TestCase): def test_PUT_bad_metadata(self): if skip: - return + raise SkipTest def put(url, token, parsed, conn, name, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) @@ -240,7 +263,7 @@ class TestContainer(unittest.TestCase): def test_POST_bad_metadata(self): if skip: - return + raise SkipTest def post(url, token, parsed, conn, extra_headers): headers = {'X-Auth-Token': token} headers.update(extra_headers) @@ -297,6 +320,204 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEquals(resp.status, 400) + def test_public_container(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 204) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + + def test_cross_account_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': 'test2', + 'X-Container-Write': 'test2'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now use the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can't access the container with the second account again + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + def test_cross_account_public_container(self): + if skip or skip2: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container completely public + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the second account + def put2(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Write': 'test2'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the second account + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 201) + + def test_nonadmin_user(self): + if skip or skip3: + raise SkipTest + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the third account + def get3(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the third account + def put3(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_user[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the third account + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + if __name__ == '__main__': unittest.main() diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py new file mode 100644 index 0000000000..4a12bf088f --- /dev/null +++ b/test/functionalnosetests/test_object.py @@ -0,0 +1,91 @@ +#!/usr/bin/python + +import unittest +from nose import SkipTest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip + + +class TestObject(unittest.TestCase): + + def setUp(self): + if skip: + raise SkipTest + self.container = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + self.obj = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + self.obj), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + raise SkipTest + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container, + self.obj), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_public_object(self): + if skip: + raise SkipTest + def get(url, token, parsed, conn): + conn.request('GET', + '%s/%s/%s' % (parsed.path, self.container, self.obj)) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.r:*'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 200) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py index 97397084e0..10eba803bc 100755 --- a/test/probe/test_account_failures.py +++ b/test/probe/test_account_failures.py @@ -39,8 +39,10 @@ class TestAccountFailures(unittest.TestCase): client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) - self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '0') + self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: @@ -56,8 +58,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') - self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '0') + self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: @@ -73,8 +77,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) get_to_final_state() - containers = client.get_account(self.url, self.token) - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -94,8 +100,10 @@ class TestAccountFailures(unittest.TestCase): client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -115,8 +123,10 @@ class TestAccountFailures(unittest.TestCase): 'once'])) for p in ps: p.wait() - self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '2') + self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: @@ -134,10 +144,12 @@ class TestAccountFailures(unittest.TestCase): '/etc/swift/account-server/%d.conf' % ((anodes[0]['port'] - 6002) / 10)]).pid sleep(2) - # This is the earlier object count and bytes because the first node - # doesn't have the newest udpates yet. - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) - containers = client.get_account(self.url, self.token) + # This is the earlier counts and bytes because the first node doesn't + # have the newest udpates yet. + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -155,8 +167,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) get_to_final_state() - containers = client.get_account(self.url, self.token) - self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '2') + self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index 932f821c1c..9c497ca88b 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -40,23 +40,23 @@ class TestContainerFailures(unittest.TestCase): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.pids[self.port2server[cnodes[0]['port']]] = \ Popen(['swift-container-server', @@ -64,11 +64,11 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[0]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -87,7 +87,7 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account @@ -95,44 +95,44 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_second_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.pids[self.port2server[cnodes[1]['port']]] = \ Popen(['swift-container-server', @@ -140,9 +140,9 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[1]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -161,42 +161,42 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): @@ -204,9 +204,9 @@ class TestContainerFailures(unittest.TestCase): client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ @@ -215,11 +215,11 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -238,7 +238,7 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account @@ -246,35 +246,35 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_last_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in (1, 2): @@ -282,9 +282,9 @@ class TestContainerFailures(unittest.TestCase): client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) for x in (1, 2): self.pids[self.port2server[cnodes[x]['port']]] = \ @@ -293,9 +293,9 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -314,29 +314,29 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) if __name__ == '__main__': diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 4b4c3a49b7..a5d5852c68 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container(cnode, cpart, - self.account, container)) + self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-object-updater', @@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): for p in ps: p.wait() objs = [o['name'] for o in direct_client.direct_get_container(cnode, - cpart, self.account, container)] + cpart, self.account, container)[1]] self.assert_(obj in objs) diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 20a29772c4..006f0d3a1e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -75,13 +75,13 @@ class TestObjectHandoff(unittest.TestCase): raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in - client.get_container(self.url, self.token, container)] + client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, - self.account, container)] + self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % @@ -126,11 +126,10 @@ class TestObjectHandoff(unittest.TestCase): kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.post_object(self.url, self.token, container, obj, - {'probe': 'value'}) - ometadata = client.head_object( - self.url, self.token, container, obj)[-1] - if ometadata.get('probe') != 'value': - raise Exception('Metadata incorrect, was %s' % repr(ometadata)) + headers={'x-object-meta-probe': 'value'}) + oheaders = client.head_object(self.url, self.token, container, obj) + if oheaders.get('x-object-meta-probe') != 'value': + raise Exception('Metadata incorrect, was %s' % repr(oheaders)) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, @@ -145,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase): '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') == 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') == 'value': raise Exception('Previously downed object server had the new ' 'metadata when it should not have it') # Run the extra server last so it'll remove it's extra partition @@ -161,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase): call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') != 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') != 'value': raise Exception( 'Previously downed object server did not have the new metadata') @@ -177,13 +176,13 @@ class TestObjectHandoff(unittest.TestCase): if not exc: raise Exception('Regular object HEAD was still successful') objs = [o['name'] for o in - client.get_container(self.url, self.token, container)] + client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( - cnode, cpart, self.account, container)] + cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % diff --git a/test/probe/test_running_with_each_type_down.py b/test/probe/test_running_with_each_type_down.py index fada1b18ac..7f2352d6ce 100755 --- a/test/probe/test_running_with_each_type_down.py +++ b/test/probe/test_running_with_each_type_down.py @@ -56,16 +56,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase): pass client.put_object(self.url, self.token, 'container1', 'object1', '1234') get_to_final_state() - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) @@ -84,15 +87,18 @@ class TestRunningWithEachTypeDown(unittest.TestCase): '/etc/swift/object-server/%d.conf' % ((onodes[0]['port'] - 6000) / 10)]).pid sleep(2) - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True # The account node was previously down. self.assert_(not found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) @@ -101,16 +107,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase): self.assert_(found1) get_to_final_state() - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 31495401f2..791fc3853f 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -21,10 +21,11 @@ from StringIO import StringIO from uuid import uuid4 from logging import StreamHandler +import sqlite3 from webob import Request from swift.auth import server as auth_server -from swift.common.db import DatabaseConnectionError +from swift.common.db import DatabaseConnectionError, get_db_connection from swift.common.utils import get_logger @@ -106,38 +107,25 @@ class TestAuthServer(unittest.TestCase): def test_validate_token_non_existant_token(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - self.assertEquals(self.controller.validate_token(token + 'bad', - cfaccount), False) - - def test_validate_token_non_existant_cfaccount(self): - auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - self.assertEquals(self.controller.validate_token(token, - cfaccount + 'bad'), False) + self.assertEquals(self.controller.validate_token(token + 'bad'), False) def test_validate_token_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_validate_token_expired(self): @@ -145,40 +133,38 @@ class TestAuthServer(unittest.TestCase): try: auth_server.time = lambda: 1 auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account('test', 'tester', + cfaccount = self.controller.create_user('test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token( - token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) auth_server.time = lambda: 1 + self.controller.token_life - self.assertEquals(self.controller.validate_token( - token, cfaccount), False) + self.assertEquals(self.controller.validate_token(token), False) finally: auth_server.time = orig_time - def test_create_account_no_new_account(self): + def test_create_user_no_new_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('', 'tester', 'testing') + result = self.controller.create_user('', 'tester', 'testing') self.assertFalse(result) - def test_create_account_no_new_user(self): + def test_create_user_no_new_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('test', '', 'testing') + result = self.controller.create_user('test', '', 'testing') self.assertFalse(result) - def test_create_account_no_new_password(self): + def test_create_user_no_new_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('test', 'tester', '') + result = self.controller.create_user('test', 'tester', '') self.assertFalse(result) - def test_create_account_good(self): + def test_create_user_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') self.assert_(url) self.assertEquals('/'.join(url.split('/')[:-1]), self.controller.default_cluster_url.rstrip('/'), repr(url)) @@ -191,7 +177,7 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_one(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test', 'tester', 'testing') + self.controller.create_user('test', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '1', repr(rv)) @@ -199,13 +185,13 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_several(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test1', 'tester', 'testing') + self.controller.create_user('test1', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test2', 'tester', 'testing') + self.controller.create_user('test2', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test3', 'tester', 'testing') + self.controller.create_user('test3', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test4', 'tester', 'testing') + self.controller.create_user('test4', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201, 201, 201, 201, 201, 201, 201, @@ -216,7 +202,7 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_one_fail(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') cfaccount = url.split('/')[-1] auth_server.http_connect = fake_http_connect(500, 500, 500) rv = self.controller.recreate_accounts() @@ -226,16 +212,16 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_several_fail(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test1', 'tester', 'testing') + url = self.controller.create_user('test1', 'tester', 'testing') cfaccounts = [url.split('/')[-1]] auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test2', 'tester', 'testing') + url = self.controller.create_user('test2', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test3', 'tester', 'testing') + url = self.controller.create_user('test3', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test4', 'tester', 'testing') + url = self.controller.create_user('test4', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(500, 500, 500, 500, 500, 500, @@ -244,20 +230,20 @@ class TestAuthServer(unittest.TestCase): rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '4', repr(rv)) failed = rv.split('[', 1)[-1][:-1].split(', ') - self.assertEquals(failed, [repr(a) for a in cfaccounts]) + self.assertEquals(set(failed), set(repr(a) for a in cfaccounts)) def test_recreate_accounts_several_fail_some(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test1', 'tester', 'testing') + url = self.controller.create_user('test1', 'tester', 'testing') cfaccounts = [url.split('/')[-1]] auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test2', 'tester', 'testing') + url = self.controller.create_user('test2', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test3', 'tester', 'testing') + url = self.controller.create_user('test3', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test4', 'tester', 'testing') + url = self.controller.create_user('test4', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(500, 500, 500, 201, 201, 201, @@ -266,11 +252,8 @@ class TestAuthServer(unittest.TestCase): rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '4', repr(rv)) failed = rv.split('[', 1)[-1][:-1].split(', ') - expected = [] - for i, value in enumerate(cfaccounts): - if not i % 2: - expected.append(repr(value)) - self.assertEquals(failed, expected) + self.assertEquals( + len(set(repr(a) for a in cfaccounts) - set(failed)), 2) def test_auth_bad_path(self): self.assertRaises(ValueError, self.controller.handle_auth, @@ -281,7 +264,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_missing_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -297,7 +280,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/testbad/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -312,7 +295,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -327,7 +310,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -342,31 +325,31 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_SOSO_good_Mosso_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_SOSO_bad_Mosso_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -386,7 +369,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_missing_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'})) @@ -402,7 +385,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_header_format(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -417,7 +400,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -432,7 +415,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -447,7 +430,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -462,26 +445,26 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_Mosso_good_SOSO_header_names(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, headers={'X-Storage-User': 'test:tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_basic_logging(self): @@ -491,10 +474,10 @@ class TestAuthServer(unittest.TestCase): logger.logger.addHandler(log_handler) try: auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') self.assertEquals(log.getvalue().rsplit(' ', 1)[0], - "auth SUCCESS create_account('test', 'tester', _) = %s" % - repr(url)) + "auth SUCCESS create_user('test', 'tester', _, False) = %s" + % repr(url)) log.truncate(0) def start_response(*args): pass @@ -594,6 +577,58 @@ class TestAuthServer(unittest.TestCase): auth_server.Request = orig_Request logger.logger.handlers.remove(log_handler) + def test_upgrading_from_db1(self): + swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex + os.mkdir(swift_dir) + try: + # Create db1 + db_file = os.path.join(swift_dir, 'auth.db') + conn = get_db_connection(db_file, okay_to_create=True) + conn.execute('''CREATE TABLE IF NOT EXISTS account ( + account TEXT, url TEXT, cfaccount TEXT, + user TEXT, password TEXT)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account + ON account (account)''') + conn.execute('''CREATE TABLE IF NOT EXISTS token ( + cfaccount TEXT, token TEXT, created FLOAT)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount + ON token (cfaccount)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created + ON token (created)''') + conn.execute('''INSERT INTO account + (account, url, cfaccount, user, password) + VALUES ('act', 'url', 'cfa', 'usr', 'pas')''') + conn.execute('''INSERT INTO token (cfaccount, token, created) + VALUES ('cfa', 'tok', '1')''') + conn.commit() + conn.close() + # Upgrade to current db + conf = {'swift_dir': swift_dir} + controller = auth_server.AuthController(conf, FakeRing()) + # Check new items exist and are correct + conn = get_db_connection(db_file) + row = conn.execute('SELECT admin FROM account').fetchone() + self.assertEquals(row[0], 't') + row = conn.execute('SELECT user FROM token').fetchone() + self.assert_(not row) + finally: + rmtree(swift_dir) + + def test_create_user_twice(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_user('test', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.assertEquals( + self.controller.create_user('test', 'tester', 'testing'), + 'already exists') + + def test_create_2users_1account(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_user('test', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + url2 = self.controller.create_user('test', 'tester2', 'testing2') + self.assertEquals(url, url2) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_acl.py b/test/unit/common/middleware/test_acl.py new file mode 100644 index 0000000000..03e76ce621 --- /dev/null +++ b/test/unit/common/middleware/test_acl.py @@ -0,0 +1,126 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from swift.common.middleware import acl + + +class TestACL(unittest.TestCase): + + def test_clean_acl(self): + value = acl.clean_acl('header', '.r:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.r:specific.host') + self.assertEquals(value, '.r:specific.host') + value = acl.clean_acl('header', '.r:.ending.with') + self.assertEquals(value, '.r:.ending.with') + value = acl.clean_acl('header', '.r:*.ending.with') + self.assertEquals(value, '.r:.ending.with') + value = acl.clean_acl('header', '.r:-*.ending.with') + self.assertEquals(value, '.r:-.ending.with') + value = acl.clean_acl('header', '.r:one,.r:two') + self.assertEquals(value, '.r:one,.r:two') + value = acl.clean_acl('header', '.r:*,.r:-specific.host') + self.assertEquals(value, '.r:*,.r:-specific.host') + value = acl.clean_acl('header', '.r:*,.r:-.ending.with') + self.assertEquals(value, '.r:*,.r:-.ending.with') + value = acl.clean_acl('header', '.r:one,.r:-two') + self.assertEquals(value, '.r:one,.r:-two') + value = acl.clean_acl('header', '.r:one,.r:-two,account,account:user') + self.assertEquals(value, '.r:one,.r:-two,account,account:user') + value = acl.clean_acl('header', 'TEST_account') + self.assertEquals(value, 'TEST_account') + value = acl.clean_acl('header', '.ref:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.referer:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.referrer:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', + ' .r : one , ,, .r:two , .r : - three ') + self.assertEquals(value, '.r:one,.r:two,.r:-three') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.unknown:test') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:*.') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : * . ') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-*.') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : - * . ') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : - ') + self.assertRaises(ValueError, acl.clean_acl, 'header', + 'user , .r : - ') + self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.r:r') + + def test_parse_acl(self): + self.assertEquals(acl.parse_acl(None), ([], [])) + self.assertEquals(acl.parse_acl(''), ([], [])) + self.assertEquals(acl.parse_acl('.r:ref1'), (['ref1'], [])) + self.assertEquals(acl.parse_acl('.r:-ref1'), (['-ref1'], [])) + self.assertEquals(acl.parse_acl('account:user'), + ([], ['account:user'])) + self.assertEquals(acl.parse_acl('account'), ([], ['account'])) + self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.r:ref3,.r:-ref4'), + (['ref3', '-ref4'], ['acc1', 'acc2:usr2'])) + self.assertEquals(acl.parse_acl( + 'acc1,acc2:usr2,.r:ref3,acc3,acc4:usr4,.r:ref5,.r:-ref6'), + (['ref3', 'ref5', '-ref6'], + ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4'])) + + def test_referrer_allowed(self): + self.assert_(not acl.referrer_allowed('host', None)) + self.assert_(not acl.referrer_allowed('host', [])) + self.assert_(acl.referrer_allowed(None, ['*'])) + self.assert_(acl.referrer_allowed('', ['*'])) + self.assert_(not acl.referrer_allowed(None, ['specific.host'])) + self.assert_(not acl.referrer_allowed('', ['specific.host'])) + self.assert_(acl.referrer_allowed('http://www.example.com/index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user@www.example.com/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://www.example.com:8080/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user@www.example.com:8080/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com:8080/index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com:8080', ['.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['.example.com'])) + self.assert_(not acl.referrer_allowed('http://thief.example.com', + ['.example.com', '-thief.example.com'])) + self.assert_(not acl.referrer_allowed('http://thief.example.com', + ['*', '-thief.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['.other.com', 'www.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['-.example.com', 'www.example.com'])) + # This is considered a relative uri to the request uri, a mode not + # currently supported. + self.assert_(not acl.referrer_allowed('www.example.com', + ['.example.com'])) + self.assert_(not acl.referrer_allowed('../index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed('www.example.com', ['*'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index cd5a02c91d..e19d6d4ca9 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -94,7 +94,7 @@ class Logger(object): class FakeApp(object): def __call__(self, env, start_response): - return "OK" + return ['204 No Content'] def start_response(*args): pass @@ -102,75 +102,192 @@ def start_response(*args): class TestAuth(unittest.TestCase): def setUp(self): - self.test_auth = auth.DevAuthMiddleware( - FakeApp(), {}, FakeMemcache(), Logger()) + self.test_auth = auth.filter_factory({})(FakeApp()) def test_auth_fail(self): old_http_connect = auth.http_connect try: auth.http_connect = mock_http_connect(404) - self.assertFalse(self.test_auth.auth('a','t')) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()}, + lambda x, y: None)) + self.assert_(result.startswith('401'), result) finally: auth.http_connect = old_http_connect def test_auth_success(self): old_http_connect = auth.http_connect try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - self.assertTrue(self.test_auth.auth('a','t')) + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) finally: auth.http_connect = old_http_connect def test_auth_memcache(self): old_http_connect = auth.http_connect try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - self.assertTrue(self.test_auth.auth('a','t')) + fake_memcache = FakeMemcache() + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) auth.http_connect = mock_http_connect(404) # Should still be in memcache - self.assertTrue(self.test_auth.auth('a','t')) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + finally: + auth.http_connect = old_http_connect + + def test_auth_just_expired(self): + old_http_connect = auth.http_connect + try: + fake_memcache = FakeMemcache() + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + auth.http_connect = mock_http_connect(404) + # Should still be in memcache, but expired + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('401'), result) finally: auth.http_connect = old_http_connect def test_middleware_success(self): old_http_connect = auth.http_connect try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v/a/c/o', headers={'x-auth-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, 'OK') + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + req = Request.blank('/v/a/c/o', headers={'x-auth-token': 'AUTH_t'}) + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') finally: auth.http_connect = old_http_connect def test_middleware_no_header(self): old_http_connect = auth.http_connect try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) req = Request.blank('/v/a/c/o') - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, ['Missing Auth Token']) + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assert_(not req.remote_user, req.remote_user) finally: auth.http_connect = old_http_connect def test_middleware_storage_token(self): old_http_connect = auth.http_connect try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v/a/c/o', headers={'x-storage-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, 'OK') + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + req = Request.blank('/v/a/c/o', + headers={'x-storage-token': 'AUTH_t'}) + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') finally: auth.http_connect = old_http_connect - def test_middleware_only_version(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v', headers={'x-auth-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, ['Bad URL']) - finally: - auth.http_connect = old_http_connect + def test_authorize_bad_path(self): + req = Request.blank('/badpath') + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/badpath') + req.remote_user = 'act:usr,act,AUTH_cfa' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_account_access(self): + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act,AUTH_cfa' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_acl_group_access(self): + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act:usr' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act2' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act:usr2' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_deny_cross_reseller(self): + # Tests that cross-reseller is denied, even if ACLs/group names match + req = Request.blank('/v1/OTHER_cfa') + req.remote_user = 'act:usr,act,AUTH_cfa' + req.acl = 'act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_acl_referrer_access(self): + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = '.r:*' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.acl = '.r:.example.com' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act' + req.referer = 'http://www.example.com/index.html' + req.acl = '.r:.example.com' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.acl = '.r:*' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/AUTH_cfa') + req.acl = '.r:.example.com' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/v1/AUTH_cfa') + req.referer = 'http://www.example.com/index.html' + req.acl = '.r:.example.com' + self.assertEquals(self.test_auth.authorize(req), None) if __name__ == '__main__': diff --git a/test/unit/common/test_client.py b/test/unit/common/test_client.py index 1f67f3e020..23d5c6d848 100644 --- a/test/unit/common/test_client.py +++ b/test/unit/common/test_client.py @@ -191,7 +191,7 @@ class TestGetAccount(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_account('http://www.test.com', 'asdf') + value = c.get_account('http://www.test.com', 'asdf')[1] self.assertEquals(value, []) @@ -200,7 +200,10 @@ class TestHeadAccount(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.head_account('http://www.tests.com', 'asdf') - self.assertEquals(value, (0, 0, 0)) + # TODO: Hmm. This doesn't really test too much as it uses a fake that + # always returns the same dict. I guess it "exercises" the code, so + # I'll leave it for now. + self.assertEquals(type(value), dict) def test_server_error(self): c.http_connection = self.fake_http_connection(500) @@ -212,7 +215,7 @@ class TestGetContainer(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_container('http://www.test.com', 'asdf', 'asdf') + value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1] self.assertEquals(value, []) diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index 029791289e..a925c118bb 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -16,7 +16,6 @@ # TODO: Tests import unittest -from swift.common import direct_client class TestAuditor(unittest.TestCase): diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 639bc4ec5c..7aaee688e8 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -55,6 +55,49 @@ class TestContainerController(unittest.TestCase): """ Tear down for testing swift.object_server.ObjectController """ rmtree(self.testdir, ignore_errors=1) + def test_acl_container(self): + # Ensure no acl by default + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assert_('x-container-read' not in response.headers) + self.assert_('x-container-write' not in response.headers) + # Ensure POSTing acls works + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': '1', 'X-Container-Read': '.r:*', + 'X-Container-Write': 'account:user'}) + self.controller.POST(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assertEquals(response.headers.get('x-container-read'), '.r:*') + self.assertEquals(response.headers.get('x-container-write'), + 'account:user') + # Ensure we can clear acls on POST + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': '3', 'X-Container-Read': '', + 'X-Container-Write': ''}) + self.controller.POST(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assert_('x-container-read' not in response.headers) + self.assert_('x-container-write' not in response.headers) + # Ensure PUTing acls works + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '4', 'X-Container-Read': '.r:*', + 'X-Container-Write': 'account:user'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assertEquals(response.headers.get('x-container-read'), '.r:*') + self.assertEquals(response.headers.get('x-container-write'), + 'account:user') + def test_HEAD(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 17308c614a..048b409c63 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -19,6 +19,7 @@ import cPickle as pickle import os import sys import unittest +from nose import SkipTest from shutil import rmtree from StringIO import StringIO from time import gmtime, sleep, strftime, time @@ -64,7 +65,7 @@ class TestObjectController(unittest.TestCase): def test_POST_update_meta(self): """ Test swift.object_server.ObjectController.POST """ if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -92,7 +93,7 @@ class TestObjectController(unittest.TestCase): def test_POST_not_exist(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, @@ -114,7 +115,7 @@ class TestObjectController(unittest.TestCase): def test_POST_container_connection(self): if not self.path_to_test_xfs: - return + raise SkipTest def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): @@ -210,7 +211,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_common(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -234,7 +235,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_overwrite(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '6', @@ -267,7 +268,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_no_etag(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'text/plain'}) @@ -286,7 +287,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_user_metadata(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -314,7 +315,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_container_connection(self): if not self.path_to_test_xfs: - return + raise SkipTest def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): @@ -376,7 +377,7 @@ class TestObjectController(unittest.TestCase): def test_HEAD(self): """ Test swift.object_server.ObjectController.HEAD """ if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c') resp = self.object_controller.HEAD(req) self.assertEquals(resp.status_int, 400) @@ -443,7 +444,7 @@ class TestObjectController(unittest.TestCase): def test_GET(self): """ Test swift.object_server.ObjectController.GET """ if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c') resp = self.object_controller.GET(req) self.assertEquals(resp.status_int, 400) @@ -532,7 +533,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_match(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), @@ -586,7 +587,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_none_match(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), @@ -637,7 +638,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_modified_since(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ @@ -674,7 +675,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_unmodified_since(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ @@ -713,7 +714,7 @@ class TestObjectController(unittest.TestCase): def test_DELETE(self): """ Test swift.object_server.ObjectController.DELETE """ if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 400) @@ -840,7 +841,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put(self): if not self.path_to_test_xfs: - return + raise SkipTest listener = listen(('localhost', 0)) port = listener.getsockname()[1] killer = spawn(wsgi.server, listener, self.object_controller, @@ -866,7 +867,7 @@ class TestObjectController(unittest.TestCase): def test_max_object_name_length(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/' + ('1' * 1024), environ={'REQUEST_METHOD': 'PUT'}, @@ -887,7 +888,7 @@ class TestObjectController(unittest.TestCase): def test_disk_file_app_iter_corners(self): if not self.path_to_test_xfs: - return + raise SkipTest df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') mkdirs(df.datadir) f = open(os.path.join(df.datadir, @@ -920,7 +921,7 @@ class TestObjectController(unittest.TestCase): def test_max_upload_time(self): if not self.path_to_test_xfs: - return + raise SkipTest class SlowBody(): def __init__(self): self.sent = 0 @@ -962,7 +963,7 @@ class TestObjectController(unittest.TestCase): def test_bad_sinces(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}, @@ -988,7 +989,7 @@ class TestObjectController(unittest.TestCase): def test_content_encoding(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain', diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index ab56dc740a..face9d3787 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -19,6 +19,7 @@ import logging import os import sys import unittest +from nose import SkipTest from ConfigParser import ConfigParser from contextlib import contextmanager from cStringIO import StringIO @@ -33,6 +34,7 @@ from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen from eventlet.timeout import Timeout import simplejson from webob import Request +from webob.exc import HTTPUnauthorized from test.unit import connect_tcp, readuntil2crlfs from swift.proxy import server as proxy_server @@ -81,7 +83,7 @@ def fake_http_connect(*code_iter, **kwargs): pass if 'slow' in kwargs: headers['content-length'] = '4' - return headers + return headers.items() def read(self, amt=None): if 'slow' in kwargs: if self.sent < 4: @@ -97,7 +99,7 @@ def fake_http_connect(*code_iter, **kwargs): self.received += 1 sleep(0.1) def getheader(self, name, default=None): - return self.getheaders().get(name.lower(), default) + return dict(self.getheaders()).get(name.lower(), default) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): @@ -203,10 +205,39 @@ class TestProxyServer(unittest.TestCase): app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + app.update_request(req) resp = app.handle_request(req) self.assertEquals(resp.status_int, 500) + def test_calls_authorize_allow(self): + called = [False] + def authorize(req): + called[0] = True + with save_globals(): + proxy_server.http_connect = fake_http_connect(200) + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), container_ring=FakeRing(), + object_ring=FakeRing()) + req = Request.blank('/v1/a') + req.environ['swift.authorize'] = authorize + app.update_request(req) + resp = app.handle_request(req) + self.assert_(called[0]) + + def test_calls_authorize_deny(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), container_ring=FakeRing(), + object_ring=FakeRing()) + req = Request.blank('/v1/a') + req.environ['swift.authorize'] = authorize + app.update_request(req) + resp = app.handle_request(req) + self.assert_(called[0]) + class TestObjectController(unittest.TestCase): @@ -224,14 +255,14 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -244,7 +275,7 @@ class TestObjectController(unittest.TestCase): give_content_type=lambda content_type: self.assertEquals(content_type, expected.next())) req = Request.blank('/a/c/%s' % filename, {}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) test_content_type('test.jpg', iter(['', '', '', 'image/jpeg', 'image/jpeg', 'image/jpeg'])) @@ -261,7 +292,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) expected = str(expected) @@ -296,7 +327,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -330,7 +361,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} proxy_server.http_connect = mock_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) - req.account = 'a' + self.app.update_request(req) req.body_file = StringIO('some data') res = controller.PUT(req) expected = str(expected) @@ -347,7 +378,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Length': str(MAX_FILE_SIZE + 1), 'Content-Type': 'foo/bar'}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 413) @@ -379,7 +410,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = mock_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(str(expected))], @@ -397,7 +428,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -417,7 +448,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/a/c/o', {}) - req.account = 'a' + self.app.update_request(req) res = controller.DELETE(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -436,7 +467,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/a/c/o', {}) - req.account = 'a' + self.app.update_request(req) res = controller.HEAD(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -460,14 +491,14 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x'*256}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 202) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x'*257}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -481,14 +512,14 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', ('X-Object-Meta-'+'x'*128): 'x'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 202) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', ('X-Object-Meta-'+'x'*129): 'x'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -500,7 +531,7 @@ class TestObjectController(unittest.TestCase): headers.update({'Content-Type': 'foo/bar'}) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers=headers) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -512,7 +543,7 @@ class TestObjectController(unittest.TestCase): headers.update({'Content-Type': 'foo/bar'}) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers=headers) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -542,7 +573,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -554,7 +585,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(201, 201, 201) # obj obj obj @@ -583,7 +614,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -607,7 +638,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -649,7 +680,7 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -663,7 +694,7 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') - req.account = 'account' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 503) @@ -708,7 +739,7 @@ class TestObjectController(unittest.TestCase): def test_proxy_passes_content_type(self): with save_globals(): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = fake_http_connect(200, 200, 200) @@ -728,7 +759,7 @@ class TestObjectController(unittest.TestCase): def test_proxy_passes_content_length_on_head(self): with save_globals(): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = fake_http_connect(200, 200, 200) @@ -777,7 +808,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 200) @@ -853,7 +884,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = \ fake_http_connect(404, 404, 404, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) @@ -861,7 +892,7 @@ class TestObjectController(unittest.TestCase): fake_http_connect(404, 404, 404, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 404) @@ -874,7 +905,7 @@ class TestObjectController(unittest.TestCase): # acct cont obj obj obj req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -883,7 +914,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * MAX_META_NAME_LENGTH) : 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -891,7 +922,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)) : 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -900,7 +931,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -908,7 +939,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -918,7 +949,7 @@ class TestObjectController(unittest.TestCase): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -927,7 +958,7 @@ class TestObjectController(unittest.TestCase): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -946,7 +977,7 @@ class TestObjectController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -954,7 +985,7 @@ class TestObjectController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -964,7 +995,7 @@ class TestObjectController(unittest.TestCase): 'container', 'object') req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj @@ -974,7 +1005,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) # acct cont acct cont objc obj obj obj @@ -986,7 +1017,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) # acct cont acct cont objc obj obj obj @@ -998,7 +1029,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 503, 503, 503) # acct cont objc objc objc @@ -1009,7 +1040,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 404, 404, 404) # acct cont objc objc objc @@ -1020,7 +1051,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj @@ -1032,7 +1063,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Copy-From': '/c/o', 'X-Object-Meta-Ours': 'okay'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 201, 201, 201) # acct cont objc obj obj obj @@ -1052,7 +1083,7 @@ class TestObjectController(unittest.TestCase): 'pointing to a valid directory.\n' \ 'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \ 'system for testing.' - return + raise SkipTest testdir = \ os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked') mkdirs(testdir) @@ -1434,7 +1465,7 @@ class TestObjectController(unittest.TestCase): 'container', 'object') req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = fake_http_connect(200, 201, 201, 201, etags=[None, '68b329da9893e34099c7d8ad5cb9c940', @@ -1452,7 +1483,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '10'}, body='1234567890') - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assert_(hasattr(req, 'bytes_transferred')) self.assertEquals(req.bytes_transferred, 10) @@ -1464,7 +1495,7 @@ class TestObjectController(unittest.TestCase): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/a/c/o') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1479,7 +1510,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '10'}, body='12345') - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(req.bytes_transferred, 5) self.assert_(hasattr(req, 'client_disconnect')) @@ -1492,7 +1523,7 @@ class TestObjectController(unittest.TestCase): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/a/c/o') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 5 @@ -1509,6 +1540,73 @@ class TestObjectController(unittest.TestCase): finally: self.app.object_chunk_size = orig_object_chunk_size + def test_GET_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.GET(req) + self.assert_(called[0]) + + def test_HEAD_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', {'REQUEST_METHOD': 'HEAD'}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.HEAD(req) + self.assert_(called[0]) + + def test_POST_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'Content-Length': '5'}, body='12345') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + + def test_PUT_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '5'}, body='12345') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + + class TestContainerController(unittest.TestCase): "Test swift.proxy_server.ContainerController" @@ -1528,14 +1626,14 @@ class TestContainerController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c/', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -1547,7 +1645,7 @@ class TestContainerController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c', {}) - req.account = 'a' + self.app.update_request(req) res = controller.HEAD(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -1570,7 +1668,7 @@ class TestContainerController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -1613,7 +1711,7 @@ class TestContainerController(unittest.TestCase): fake_http_connect(200, 200, 200, 200) self.app.memcache.store = {} req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 200) @@ -1657,7 +1755,7 @@ class TestContainerController(unittest.TestCase): self.app.memcache = MockMemcache(allow_lock=True) proxy_server.http_connect = fake_http_connect(200, 200, 200, 201, 201, 201, missing_container=True) req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 201) @@ -1703,7 +1801,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c?format=json') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1715,7 +1813,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c?format=json') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 1 @@ -1760,8 +1858,7 @@ class TestContainerController(unittest.TestCase): 201, give_connect=test_connect) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={test_header: test_value}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) res = getattr(controller, method)(req) self.assertEquals(test_errors, []) @@ -1776,7 +1873,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'a', 'c') proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) @@ -1784,16 +1881,14 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1801,16 +1896,14 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1820,8 +1913,7 @@ class TestContainerController(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -1830,8 +1922,7 @@ class TestContainerController(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1849,8 +1940,7 @@ class TestContainerController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -1858,11 +1948,96 @@ class TestContainerController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) + def test_POST_calls_clean_acl(self): + called = [False] + def clean_acl(header, value): + called[0] = True + raise ValueError('fake error') + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Container-Read': '.r:*'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + called[0] = False + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Container-Write': '.r:*'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + + def test_PUT_calls_clean_acl(self): + called = [False] + def clean_acl(header, value): + called[0] = True + raise ValueError('fake error') + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Container-Read': '.r:*'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + called[0] = False + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Container-Write': '.r:*'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + + def test_GET_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.GET(req) + self.assert_(called[0]) + + def test_HEAD_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', {'REQUEST_METHOD': 'HEAD'}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.HEAD(req) + self.assert_(called[0]) + class TestAccountController(unittest.TestCase): @@ -1875,12 +2050,12 @@ class TestAccountController(unittest.TestCase): with save_globals(): proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a', {}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a/', {}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -1930,7 +2105,7 @@ class TestAccountController(unittest.TestCase): dev['port'] = 1 ## can't connect on this port controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) @@ -1941,7 +2116,7 @@ class TestAccountController(unittest.TestCase): dev['port'] = -1 ## invalid port number controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) @@ -1950,7 +2125,7 @@ class TestAccountController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/a?format=json') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1961,7 +2136,7 @@ class TestAccountController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/a?format=json') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 1 @@ -1999,7 +2174,7 @@ class TestAccountController(unittest.TestCase): give_connect=test_connect) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={test_header: test_value}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(test_errors, []) @@ -2008,7 +2183,7 @@ class TestAccountController(unittest.TestCase): controller = proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) @@ -2016,14 +2191,14 @@ class TestAccountController(unittest.TestCase): req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2031,14 +2206,14 @@ class TestAccountController(unittest.TestCase): req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2048,7 +2223,7 @@ class TestAccountController(unittest.TestCase): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) @@ -2057,7 +2232,7 @@ class TestAccountController(unittest.TestCase): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2075,7 +2250,7 @@ class TestAccountController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) @@ -2083,7 +2258,7 @@ class TestAccountController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400)