Add container-level ACLs. Refactor auth to be more pluggable. Document how to make your own auth.

bin/st

@@ -154,19 +154,26 @@ except:
 
     def get_auth(url, user, key, snet=False):
         """
-        Get authentication credentials
+        Get authentication/authorization credentials.
 
-        :param url: authentication URL
+        The snet parameter is used for Rackspace's ServiceNet internal network
-        :param user: user to auth as
+        implementation. In this function, it simply adds *snet-* to the beginning
-        :param key: key or passowrd for auth
+        of the host name for the returned storage URL. With Rackspace Cloud Files,
-        :param snet: use SERVICENET internal network default is False 
+        use of this network path causes no bandwidth charges but requires the
-        :returns: tuple of (storage URL, storage token, auth token)
+        client to be running on Rackspace's ServiceNet network.
+
+        :param url: authentication/authorization URL
+        :param user: user to authenticate as
+        :param key: key or password for authorization
+        :param snet: use SERVICENET internal network (see above), default is False
+        :returns: tuple of (storage URL, auth token)
         :raises ClientException: HTTP GET request to auth URL failed
         """
         parsed, conn = http_connection(url)
         conn.request('GET', parsed.path, '',
                      {'X-Auth-User': user, 'X-Auth-Key': key})
         resp = conn.getresponse()
+        resp.read()
         if resp.status < 200 or resp.status >= 300:
             raise ClientException('Auth GET failed', http_scheme=parsed.scheme,
                     http_host=conn.host, http_port=conn.port,
@@ -196,18 +203,21 @@ except:
                           conn object)
         :param full_listing: if True, return a full listing, else returns a max
                              of 10000 listings
-        :returns: a list of accounts
+        :returns: a tuple of (response headers, a list of containers) The response
+                  headers will be a dict and all header names will be lowercase. 
         :raises ClientException: HTTP GET request failed
         """
         if not http_conn:
             http_conn = http_connection(url)
         if full_listing:
-            rv = []
+            rv = get_account(url, token, marker, limit, prefix, http_conn)
-            listing = get_account(url, token, marker, limit, prefix, http_conn)
+            listing = rv[1]
             while listing:
-                rv.extend(listing)
                 marker = listing[-1]['name']
-                listing = get_account(url, token, marker, limit, prefix, http_conn)
+                listing = \
+                    get_account(url, token, marker, limit, prefix, http_conn)[1]
+                if listing:
+                    rv.extend(listing)
             return rv
         parsed, conn = http_conn
         qs = 'format=json'
@@ -220,6 +230,9 @@ except:
         conn.request('GET', '%s?%s' % (parsed.path, qs), '',
                      {'X-Auth-Token': token})
         resp = conn.getresponse()
+        resp_headers = {}
+        for header, value in resp.getheaders():
+            resp_headers[header.lower()] = value
         if resp.status < 200 or resp.status >= 300:
             resp.read()
             raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@@ -228,8 +241,8 @@ except:
                     http_reason=resp.reason)
         if resp.status == 204:
             resp.read()
-            return []
+            return resp_headers, []
-        return json_loads(resp.read())
+        return resp_headers, json_loads(resp.read())
 
 
     def head_account(url, token, http_conn=None):
@@ -240,7 +253,8 @@ except:
         :param token: auth token
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
-        :returns: a tuple of (container count, object count, bytes used)
+        :returns: a dict containing the response's headers (all header names will
+                  be lowercase)
         :raises ClientException: HTTP HEAD request failed
         """
         if http_conn:
@@ -249,14 +263,42 @@ except:
             parsed, conn = http_connection(url)
         conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token})
         resp = conn.getresponse()
+        resp.read()
         if resp.status < 200 or resp.status >= 300:
             raise ClientException('Account HEAD failed', http_scheme=parsed.scheme,
                     http_host=conn.host, http_port=conn.port,
                     http_path=parsed.path, http_status=resp.status,
                     http_reason=resp.reason)
-        return int(resp.getheader('x-account-container-count', 0)), \
+        resp_headers = {}
-               int(resp.getheader('x-account-object-count', 0)), \
+        for header, value in resp.getheaders():
-               int(resp.getheader('x-account-bytes-used', 0))
+            resp_headers[header.lower()] = value
+        return resp_headers
+
+
+    def post_account(url, token, headers, http_conn=None):
+        """
+        Update an account's metadata.
+
+        :param url: storage URL
+        :param token: auth token
+        :param headers: additional headers to include in the request
+        :param http_conn: HTTP connection object (If None, it will create the
+                          conn object)
+        :raises ClientException: HTTP POST request failed
+        """
+        if http_conn:
+            parsed, conn = http_conn
+        else:
+            parsed, conn = http_connection(url)
+        headers['X-Auth-Token'] = token
+        conn.request('POST', parsed.path, '', headers)
+        resp = conn.getresponse()
+        resp.read()
+        if resp.status < 200 or resp.status >= 300:
+            raise ClientException('Account POST failed',
+                    http_scheme=parsed.scheme, http_host=conn.host,
+                    http_port=conn.port, http_path=path, http_status=resp.status,
+                    http_reason=resp.reason)
 
 
     def get_container(url, token, container, marker=None, limit=None,
@@ -276,23 +318,25 @@ except:
                           conn object)
         :param full_listing: if True, return a full listing, else returns a max
                              of 10000 listings
-        :returns: a list of objects
+        :returns: a tuple of (response headers, a list of objects) The response
+                  headers will be a dict and all header names will be lowercase. 
         :raises ClientException: HTTP GET request failed
         """
         if not http_conn:
             http_conn = http_connection(url)
         if full_listing:
-            rv = []
+            rv = get_container(url, token, container, marker, limit, prefix,
-            listing = get_container(url, token, container, marker, limit, prefix,
                                delimiter, http_conn)
+            listing = rv[1]
             while listing:
-                rv.extend(listing)
                 if not delimiter:
                     marker = listing[-1]['name']
                 else:
                     marker = listing[-1].get('name', listing[-1].get('subdir'))
                 listing = get_container(url, token, container, marker, limit,
-                                        prefix, delimiter, http_conn)
+                                        prefix, delimiter, http_conn)[1]
+                if listing:
+                    rv[1].extend(listing)
             return rv
         parsed, conn = http_conn
         path = '%s/%s' % (parsed.path, quote(container))
@@ -313,10 +357,13 @@ except:
                     http_scheme=parsed.scheme, http_host=conn.host,
                     http_port=conn.port, http_path=path, http_query=qs,
                     http_status=resp.status, http_reason=resp.reason)
+        resp_headers = {}
+        for header, value in resp.getheaders():
+            resp_headers[header.lower()] = value
         if resp.status == 204:
             resp.read()
-            return []
+            return resp_headers, []
-        return json_loads(resp.read())
+        return resp_headers, json_loads(resp.read())
 
 
     def head_container(url, token, container, http_conn=None):
@@ -328,7 +375,8 @@ except:
         :param container: container name to get stats for
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
-        :returns: a tuple of (object count, bytes used)
+        :returns: a dict containing the response's headers (all header names will
+                  be lowercase)
         :raises ClientException: HTTP HEAD request failed
         """
         if http_conn:
@@ -344,17 +392,20 @@ except:
                     http_scheme=parsed.scheme, http_host=conn.host,
                     http_port=conn.port, http_path=path, http_status=resp.status,
                     http_reason=resp.reason)
-        return int(resp.getheader('x-container-object-count', 0)), \
+        resp_headers = {}
-               int(resp.getheader('x-container-bytes-used', 0))
+        for header, value in resp.getheaders():
+            resp_headers[header.lower()] = value
+        return resp_headers
 
 
-    def put_container(url, token, container, http_conn=None):
+    def put_container(url, token, container, headers=None, http_conn=None):
         """
         Create a container
 
         :param url: storage URL
         :param token: auth token
         :param container: container name to create
+        :param headers: additional headers to include in the request
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
         :raises ClientException: HTTP PUT request failed
@@ -364,7 +415,10 @@ except:
         else:
             parsed, conn = http_connection(url)
         path = '%s/%s' % (parsed.path, quote(container))
-        conn.request('PUT', path, '', {'X-Auth-Token': token})
+        if not headers:
+            headers = {}
+        headers['X-Auth-Token'] = token
+        conn.request('PUT', path, '', headers)
         resp = conn.getresponse()
         resp.read()
         if resp.status < 200 or resp.status >= 300:
@@ -374,6 +428,34 @@ except:
                     http_reason=resp.reason)
 
 
+    def post_container(url, token, container, headers, http_conn=None):
+        """
+        Update a container's metadata.
+
+        :param url: storage URL
+        :param token: auth token
+        :param container: container name to update
+        :param headers: additional headers to include in the request
+        :param http_conn: HTTP connection object (If None, it will create the
+                          conn object)
+        :raises ClientException: HTTP POST request failed
+        """
+        if http_conn:
+            parsed, conn = http_conn
+        else:
+            parsed, conn = http_connection(url)
+        path = '%s/%s' % (parsed.path, quote(container))
+        headers['X-Auth-Token'] = token
+        conn.request('POST', path, '', headers)
+        resp = conn.getresponse()
+        resp.read()
+        if resp.status < 200 or resp.status >= 300:
+            raise ClientException('Container POST failed',
+                    http_scheme=parsed.scheme, http_host=conn.host,
+                    http_port=conn.port, http_path=path, http_status=resp.status,
+                    http_reason=resp.reason)
+
+
     def delete_container(url, token, container, http_conn=None):
         """
         Delete a container
@@ -411,8 +493,12 @@ except:
         :param name: object name to get
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
-        :param resp_chunk_size: if defined, chunk size of data to read
+        :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
-        :returns: a list of objects
+                                you specify a resp_chunk_size you must fully read
+                                the object's contents before making another
+                                request.
+        :returns: a tuple of (response headers, the object's contents) The response
+                  headers will be a dict and all header names will be lowercase. 
         :raises ClientException: HTTP GET request failed
         """
         if http_conn:
@@ -427,10 +513,6 @@ except:
             raise ClientException('Object GET failed', http_scheme=parsed.scheme,
                     http_host=conn.host, http_port=conn.port, http_path=path,
                     http_status=resp.status, http_reason=resp.reason)
-        metadata = {}
-        for key, value in resp.getheaders():
-            if key.lower().startswith('x-object-meta-'):
-                metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
         if resp_chunk_size:
 
             def _object_body():
@@ -441,12 +523,10 @@ except:
             object_body = _object_body()
         else:
             object_body = resp.read()
-        return resp.getheader('content-type'), \
+        resp_headers = {}
-               int(resp.getheader('content-length', 0)), \
+        for header, value in resp.getheaders():
-               resp.getheader('last-modified'), \
+            resp_headers[header.lower()] = value
-               resp.getheader('etag').strip('"'), \
+        return resp_headers, object_body
-               metadata, \
-               object_body
 
 
     def head_object(url, token, container, name, http_conn=None):
@@ -459,8 +539,8 @@ except:
         :param name: object name to get info for
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
-        :returns: a tuple of (content type, content length, last modfied, etag,
+        :returns: a dict containing the response's headers (all header names will
-                  dictionary of metadata)
+                  be lowercase)
         :raises ClientException: HTTP HEAD request failed
         """
         if http_conn:
@@ -475,20 +555,15 @@ except:
             raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
                     http_host=conn.host, http_port=conn.port, http_path=path,
                     http_status=resp.status, http_reason=resp.reason)
-        metadata = {}
+        resp_headers = {}
-        for key, value in resp.getheaders():
+        for header, value in resp.getheaders():
-            if key.lower().startswith('x-object-meta-'):
+            resp_headers[header.lower()] = value
-                metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
+        return resp_headers
-        return resp.getheader('content-type'), \
-               int(resp.getheader('content-length', 0)), \
-               resp.getheader('last-modified'), \
-               resp.getheader('etag').strip('"'), \
-               metadata
 
 
-    def put_object(url, token, container, name, contents, metadata={},
+    def put_object(url, token, container, name, contents, content_length=None,
-                   content_length=None, etag=None, chunk_size=65536,
+                   etag=None, chunk_size=65536, content_type=None, headers=None,
-                   content_type=None, http_conn=None):
+                   http_conn=None):
         """
         Put an object
 
@@ -496,12 +571,12 @@ except:
         :param token: auth token
         :param container: container name that the object is in
         :param name: object name to put
-        :param contents: file like object to read object data from
+        :param contents: a string or a file like object to read object data from
-        :param metadata: dictionary of object metadata
         :param content_length: value to send as content-length header
         :param etag: etag of contents
         :param chunk_size: chunk size of data to write
         :param content_type: value to send as content-type header
+        :param headers: additional headers to include in the request
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
         :returns: etag from server response
@@ -512,9 +587,9 @@ except:
         else:
             parsed, conn = http_connection(url)
         path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
-        headers = {'X-Auth-Token': token}
+        if not headers:
-        for key, value in metadata.iteritems():
+            headers = {}
-            headers['X-Object-Meta-%s' % quote(key)] = quote(value)
+        headers['X-Auth-Token'] = token
         if etag:
             headers['ETag'] = etag.strip('"')
         if content_length is not None:
@@ -550,15 +625,15 @@ except:
         return resp.getheader('etag').strip('"')
 
 
-    def post_object(url, token, container, name, metadata, http_conn=None):
+    def post_object(url, token, container, name, headers, http_conn=None):
         """
-        Change object metadata
+        Update object metadata
 
         :param url: storage URL
         :param token: auth token
         :param container: container name that the object is in
-        :param name: object name to change
+        :param name: name of the object to update
-        :param metadata: dictionary of object metadata
+        :param headers: additional headers to include in the request
         :param http_conn: HTTP connection object (If None, it will create the
                           conn object)
         :raises ClientException: HTTP POST request failed
@@ -568,9 +643,7 @@ except:
         else:
             parsed, conn = http_connection(url)
         path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
-        headers = {'X-Auth-Token': token}
+        headers['X-Auth-Token'] = token
-        for key, value in metadata.iteritems():
-            headers['X-Object-Meta-%s' % quote(key)] = quote(value)
         conn.request('POST', path, '', headers)
         resp = conn.getresponse()
         resp.read()
@@ -632,19 +705,23 @@ except:
             self.attempts = 0
             self.snet = snet
 
+        def get_auth(self):
+            return get_auth(self.authurl, self.user, self.key, snet=self.snet)
+
+        def http_connection(self):
+            return http_connection(self.url)
+
         def _retry(self, func, *args, **kwargs):
-            kwargs['http_conn'] = self.http_conn
             self.attempts = 0
             backoff = 1
             while self.attempts <= self.retries:
                 self.attempts += 1
                 try:
                     if not self.url or not self.token:
-                        self.url, self.token = \
+                        self.url, self.token = self.get_auth()
-                            get_auth(self.authurl, self.user, self.key, snet=self.snet)
                         self.http_conn = None
                     if not self.http_conn:
-                        self.http_conn = http_connection(self.url)
+                        self.http_conn = self.http_connection()
                     kwargs['http_conn'] = self.http_conn
                     rv = func(self.url, self.token, *args, **kwargs)
                     return rv
@@ -667,63 +744,71 @@ except:
                 backoff *= 2
 
         def head_account(self):
-            """Wrapper for head_account"""
+            """Wrapper for :func:`head_account`"""
             return self._retry(head_account)
 
         def get_account(self, marker=None, limit=None, prefix=None,
                         full_listing=False):
-            """Wrapper for get_account"""
+            """Wrapper for :func:`get_account`"""
-            # TODO: With full_listing=True this will restart the entire listing
+            # TODO(unknown): With full_listing=True this will restart the entire
-            # with each retry. Need to make a better version that just retries
+            # listing with each retry. Need to make a better version that just
-            # where it left off.
+            # retries where it left off.
             return self._retry(get_account, marker=marker, limit=limit,
                                prefix=prefix, full_listing=full_listing)
 
+        def post_account(self, headers):
+            """Wrapper for :func:`post_account`"""
+            return self._retry(post_account, headers)
+
         def head_container(self, container):
-            """Wrapper for head_container"""
+            """Wrapper for :func:`head_container`"""
             return self._retry(head_container, container)
 
         def get_container(self, container, marker=None, limit=None, prefix=None,
                           delimiter=None, full_listing=False):
-            """Wrapper for get_container"""
+            """Wrapper for :func:`get_container`"""
-            # TODO: With full_listing=True this will restart the entire listing
+            # TODO(unknown): With full_listing=True this will restart the entire
-            # with each retry. Need to make a better version that just retries
+            # listing with each retry. Need to make a better version that just
-            # where it left off.
+            # retries where it left off.
             return self._retry(get_container, container, marker=marker,
                                limit=limit, prefix=prefix, delimiter=delimiter,
                                full_listing=full_listing)
 
-        def put_container(self, container):
+        def put_container(self, container, headers=None):
-            """Wrapper for put_container"""
+            """Wrapper for :func:`put_container`"""
-            return self._retry(put_container, container)
+            return self._retry(put_container, container, headers=headers)
+
+        def post_container(self, container, headers):
+            """Wrapper for :func:`post_container`"""
+            return self._retry(post_container, container, headers)
 
         def delete_container(self, container):
-            """Wrapper for delete_container"""
+            """Wrapper for :func:`delete_container`"""
             return self._retry(delete_container, container)
 
         def head_object(self, container, obj):
-            """Wrapper for head_object"""
+            """Wrapper for :func:`head_object`"""
             return self._retry(head_object, container, obj)
 
         def get_object(self, container, obj, resp_chunk_size=None):
-            """Wrapper for get_object"""
+            """Wrapper for :func:`get_object`"""
             return self._retry(get_object, container, obj,
                                resp_chunk_size=resp_chunk_size)
 
-        def put_object(self, container, obj, contents, metadata={},
+        def put_object(self, container, obj, contents, content_length=None,
-                       content_length=None, etag=None, chunk_size=65536,
+                       etag=None, chunk_size=65536, content_type=None,
-                       content_type=None):
+                       headers=None):
-            """Wrapper for put_object"""
+            """Wrapper for :func:`put_object`"""
             return self._retry(put_object, container, obj, contents,
-                metadata=metadata, content_length=content_length, etag=etag,
+                content_length=content_length, etag=etag, chunk_size=chunk_size,
-                chunk_size=chunk_size, content_type=content_type)
+                content_type=content_type, headers=headers)
 
-        def post_object(self, container, obj, metadata):
+        def post_object(self, container, obj, headers):
-            """Wrapper for post_object"""
+            """Wrapper for :func:`post_object`"""
-            return self._retry(post_object, container, obj, metadata)
+            return self._retry(post_object, container, obj, headers)
 
         def delete_object(self, container, obj):
-            """Wrapper for delete_object"""
+            """Wrapper for :func:`delete_object`"""
             return self._retry(delete_object, container, obj)
 
     # End inclusion of swift.common.client
@@ -805,7 +890,7 @@ def st_delete(options, args):
             marker = ''
             while True:
                 objects = [o['name'] for o in
-                           conn.get_container(container, marker=marker)]
+                           conn.get_container(container, marker=marker)[1]]
                 if not objects:
                     break
                 for obj in objects:
@@ -847,7 +932,7 @@ def st_delete(options, args):
             marker = ''
             while True:
                 containers = \
-                    [c['name'] for c in conn.get_account(marker=marker)]
+                    [c['name'] for c in conn.get_account(marker=marker)[1]]
                 if not containers:
                     break
                 for container in containers:
@@ -893,8 +978,11 @@ def st_download(options, args):
     object_queue = Queue(10000)
     def _download_object((container, obj), conn):
         try:
-            content_type, content_length, _, etag, metadata, body = \
+            headers, body = \
                 conn.get_object(container, obj, resp_chunk_size=65536)
+            content_type = headers.get('content-type')
+            content_length = int(headers.get('content-length'))
+            etag = headers.get('etag')
             path = options.yes_all and join(container, obj) or obj
             if path[:1] in ('/', '\\'):
                 path = path[1:]
@@ -925,8 +1013,8 @@ def st_download(options, args):
                 options.error_queue.put(
                     '%s: read_length != content_length, %d != %d' %
                     (path, read_length, content_length))
-            if 'mtime' in metadata:
+            if 'x-object-meta-mtime' in headers:
-                mtime = float(metadata['mtime'])
+                mtime = float(headers['x-object-meta-mtime'])
                 utime(path, (mtime, mtime))
             if options.verbose:
                 options.print_queue.put(path)
@@ -941,7 +1029,7 @@ def st_download(options, args):
             marker = ''
             while True:
                 objects = [o['name'] for o in
-                           conn.get_container(container, marker=marker)]
+                           conn.get_container(container, marker=marker)[1]]
                 if not objects:
                     break
                 for obj in objects:
@@ -969,7 +1057,7 @@ def st_download(options, args):
             marker = ''
             while True:
                 containers = [c['name']
-                              for c in conn.get_account(marker=marker)]
+                              for c in conn.get_account(marker=marker)[1]]
                 if not containers:
                     break
                 for container in containers:
@@ -1016,10 +1104,11 @@ def st_list(options, args):
         marker = ''
         while True:
             if not args:
-                items = conn.get_account(marker=marker, prefix=options.prefix)
+                items = \
+                    conn.get_account(marker=marker, prefix=options.prefix)[1]
             else:
                 items = conn.get_container(args[0], marker=marker,
-                            prefix=options.prefix, delimiter=options.delimiter)
+                    prefix=options.prefix, delimiter=options.delimiter)[1]
             if not items:
                 break
             for item in items:
@@ -1042,46 +1131,85 @@ def st_stat(options, args):
     conn = Connection(options.auth, options.user, options.key)
     if not args:
         try:
-            container_count, object_count, bytes_used = conn.head_account()
+            headers = conn.head_account()
+            container_count = int(headers.get('x-account-container-count', 0))
+            object_count = int(headers.get('x-account-object-count', 0))
+            bytes_used = int(headers.get('x-account-bytes-used', 0))
             options.print_queue.put('''
    Account: %s
 Containers: %d
    Objects: %d
      Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count,
                                  object_count, bytes_used))
+            for key, value in headers.items():
+                if key.startswith('x-account-meta-'):
+                    options.print_queue.put('%10s: %s' % ('Meta %s' %
+                        key[len('x-account-meta-'):].title(), value))
+            for key, value in headers.items():
+                if not key.startswith('x-account-meta-') and key not in (
+                        'content-length', 'date', 'x-account-container-count',
+                        'x-account-object-count', 'x-account-bytes-used'):
+                    options.print_queue.put(
+                        '%10s: %s' % (key.title(), value))
         except ClientException, err:
             if err.http_status != 404:
                 raise
             options.error_queue.put('Account not found')
     elif len(args) == 1:
         try:
-            object_count, bytes_used = conn.head_container(args[0])
+            headers = conn.head_container(args[0])
+            object_count = int(headers.get('x-container-object-count', 0))
+            bytes_used = int(headers.get('x-container-bytes-used', 0))
             options.print_queue.put('''
   Account: %s
 Container: %s
   Objects: %d
-    Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
+    Bytes: %d
-                                object_count, bytes_used))
+ Read ACL: %s
+Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
+                                object_count, bytes_used,
+                                headers.get('x-container-read', ''),
+                                headers.get('x-container-write', '')))
+            for key, value in headers.items():
+                if key.startswith('x-container-meta-'):
+                    options.print_queue.put('%9s: %s' % ('Meta %s' %
+                        key[len('x-container-meta-'):].title(), value))
+            for key, value in headers.items():
+                if not key.startswith('x-container-meta-') and key not in (
+                        'content-length', 'date', 'x-container-object-count',
+                        'x-container-bytes-used', 'x-container-read',
+                        'x-container-write'):
+                    options.print_queue.put(
+                        '%9s: %s' % (key.title(), value))
         except ClientException, err:
             if err.http_status != 404:
                 raise
             options.error_queue.put('Container %s not found' % repr(args[0]))
     elif len(args) == 2:
         try:
-            content_type, content_length, last_modified, etag, metadata = \
+            headers = conn.head_object(args[0], args[1])
-                conn.head_object(args[0], args[1])
             options.print_queue.put('''
        Account: %s
      Container: %s
         Object: %s
   Content Type: %s
-Content Length: %d
+Content Length: %s
  Last Modified: %s
           ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0],
-                                     args[1], content_type, content_length,
+                                     args[1], headers.get('content-type'),
-                                     last_modified, etag))
+                                     headers.get('content-length'),
-            for key, value in metadata.items():
+                                     headers.get('last-modified'),
-                options.print_queue.put('%14s: %s' % ('Meta %s' % key, value))
+                                     headers.get('etag')))
+            for key, value in headers.items():
+                if key.startswith('x-object-meta-'):
+                    options.print_queue.put('%14s: %s' % ('Meta %s' %
+                        key[len('x-object-meta-'):].title(), value))
+            for key, value in headers.items():
+                if not key.startswith('x-object-meta-') and key not in (
+                        'content-type', 'content-length', 'last-modified',
+                        'etag', 'date'):
+                    options.print_queue.put(
+                        '%14s: %s' % (key.title(), value))
         except ClientException, err:
             if err.http_status != 404:
                 raise
@@ -1092,6 +1220,63 @@ Content Length: %d
                                 (basename(argv[0]), st_stat_help))
 
 
+st_post_help = '''
+post [options] [container] [object]
+    Updates meta information for the account, container, or object depending on
+    the args given. If the container is not found, it will be created
+    automatically; but this is not true for accounts and objects. Containers
+    also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
+    or --meta option is allowed on all and used to define the user meta data
+    items to set in the form Name:Value. This option can be repeated. Example:
+    post -m Color:Blue -m Size:Large'''.strip('\n')
+def st_post(options, args):
+    conn = Connection(options.auth, options.user, options.key)
+    if not args:
+        headers = {}
+        for item in options.meta:
+            split_item = item.split(':')
+            headers['X-Account-Meta-' + split_item[0]] = \
+                len(split_item) > 1 and split_item[1]
+        try:
+            conn.post_account(headers=headers)
+        except ClientException, err:
+            if err.http_status != 404:
+                raise
+            options.error_queue.put('Account not found')
+    elif len(args) == 1:
+        headers = {}
+        for item in options.meta:
+            split_item = item.split(':')
+            headers['X-Container-Meta-' + split_item[0]] = \
+                len(split_item) > 1 and split_item[1]
+        if options.read_acl is not None:
+            headers['X-Container-Read'] = options.read_acl
+        if options.write_acl is not None:
+            headers['X-Container-Write'] = options.write_acl
+        try:
+            conn.post_container(args[0], headers=headers)
+        except ClientException, err:
+            if err.http_status != 404:
+                raise
+            conn.put_container(args[0], headers=headers)
+    elif len(args) == 2:
+        headers = {}
+        for item in options.meta:
+            split_item = item.split(':')
+            headers['X-Object-Meta-' + split_item[0]] = \
+                len(split_item) > 1 and split_item[1]
+        try:
+            conn.post_object(args[0], args[1], headers=headers)
+        except ClientException, err:
+            if err.http_status != 404:
+                raise
+            options.error_queue.put('Object %s not found' %
+                                    repr('%s/%s' % (args[0], args[1])))
+    else:
+        options.error_queue.put('Usage: %s [options] %s' %
+                                (basename(argv[0]), st_post_help))
+
+
 st_upload_help = '''
 upload [options] container file_or_directory [file_or_directory] [...]
     Uploads to the given container the files and directories specified by the
@@ -1108,35 +1293,41 @@ def st_upload(options, args):
             obj = path
             if obj.startswith('./') or obj.startswith('.\\'):
                 obj = obj[2:]
-            metadata = {'mtime': str(getmtime(path))}
+            put_headers = {'x-object-meta-mtime': str(getmtime(path))}
             if dir_marker:
                 if options.changed:
                     try:
-                        ct, cl, lm, et, md = conn.head_object(args[0], obj)
+                        headers = conn.head_object(args[0], obj)
+                        ct = headers.get('content-type')
+                        cl = int(headers.get('content-length'))
+                        et = headers.get('etag')
+                        mt = headers.get('x-object-meta-mtime')
                         if ct.split(';', 1)[0] == 'text/directory' and \
                                 cl == 0 and \
                                 et == 'd41d8cd98f00b204e9800998ecf8427e' and \
-                                md.get('mtime') == metadata['mtime']:
+                                mt == put_headers['x-object-meta-mtime']:
                             return
                     except ClientException, err:
                         if err.http_status != 404:
                             raise
                 conn.put_object(args[0], obj, '', content_length=0,
                                 content_type='text/directory',
-                                metadata=metadata)
+                                headers=put_headers)
             else:
                 if options.changed:
                     try:
-                        ct, cl, lm, et, md = conn.head_object(args[0], obj)
+                        headers = conn.head_object(args[0], obj)
+                        cl = int(headers.get('content-length'))
+                        mt = headers.get('x-object-meta-mtime')
                         if cl == getsize(path) and \
-                                md.get('mtime') == metadata['mtime']:
+                                mt == put_headers['x-object-meta-mtime']:
                             return
                     except ClientException, err:
                         if err.http_status != 404:
                             raise
                 conn.put_object(args[0], obj, open(path, 'rb'),
                                 content_length=getsize(path),
-                                metadata=metadata)
+                                headers=put_headers)
             if options.verbose:
                 options.print_queue.put(obj)
         except OSError, err:
@@ -1163,8 +1354,15 @@ def st_upload(options, args):
     for thread in file_threads:
         thread.start()
     conn = create_connection()
+    # Try to create the container, just in case it doesn't exist. If this
+    # fails, it might just be because the user doesn't have container PUT
+    # permissions, so we'll ignore any error. If there's really a problem,
+    # it'll surface on the first object PUT.
     try:
         conn.put_container(args[0])
+    except:
+        pass
+    try:
         for arg in args[1:]:
             if isdir(arg):
                 _upload_dir(arg)
@@ -1190,6 +1388,7 @@ Commands:
   %(st_stat_help)s
   %(st_list_help)s
   %(st_upload_help)s
+  %(st_post_help)s
   %(st_download_help)s
   %(st_delete_help)s
 
@@ -1215,6 +1414,17 @@ Example:
                       help='For the list command on containers: will roll up '
                       'items with the given delimiter (see Cloud Files '
                       'general documentation for what this means).')
+    parser.add_option('-r', '--read-acl', dest='read_acl',
+                      help='Sets the Read ACL with post container commands. '
+                      'Quick summary of ACL syntax: .r:*, .r:-.example.com, '
+                      '.r:www.example.com, account1, account2:user2')
+    parser.add_option('-w', '--write-acl', dest='write_acl',
+                      help='Sets the Write ACL with post container commands. '
+                      'Quick summary of ACL syntax: account1, account2:user2')
+    parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
+                      help='Sets a meta data item of the syntax name:value '
+                      'for use with post commands. This option may be '
+                      'repeated. Example: -m Color:Blue -m Size:Large')
     parser.add_option('-A', '--auth', dest='auth',
                       help='URL for obtaining an auth token')
     parser.add_option('-U', '--user', dest='user',
@@ -1235,7 +1445,7 @@ overridden with -A, -U, or -K.'''.strip('\n')
             if not getattr(options, attr, None):
                 exit(required_help)
 
-    commands = ('delete', 'download', 'list', 'stat', 'upload')
+    commands = ('delete', 'download', 'list', 'post', 'stat', 'upload')
     if not args or args[0] not in commands:
         parser.print_usage()
         if args:

bin/swift-auth-add-user

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ConfigParser import ConfigParser
+from optparse import OptionParser
+from os.path import basename
+from sys import argv, exit
+
+from swift.common.bufferedhttp import http_connect_raw as http_connect
+
+
+if __name__ == '__main__':
+    default_conf = '/etc/swift/auth-server.conf'
+    parser = OptionParser(
+        usage='Usage: %prog [options] <account> <user> <password>')
+    parser.add_option('-c', '--conf', dest='conf', default=default_conf,
+        help='Configuration file to determine how to connect to the local '
+        'auth server (default: %s).' % default_conf)
+    parser.add_option('-a', '--admin', dest='admin', action='store_true',
+        default=False, help='Give the user administrator access; otherwise '
+        'the user will only have access to containers specifically allowed '
+        'with ACLs.')
+    args = argv[1:]
+    if not args:
+        args.append('-h')
+    (options, args) = parser.parse_args(args)
+    if len(args) != 3:
+        parser.parse_args(['-h'])
+    account, user, password = args
+    c = ConfigParser()
+    if not c.read(options.conf):
+        exit('Unable to read conf file: %s' % options.conf)
+    conf = dict(c.items('app:auth-server'))
+    host = conf.get('bind_ip', '127.0.0.1')
+    port = int(conf.get('bind_port', 11000))
+    ssl = conf.get('cert_file') is not None
+    path = '/account/%s/%s' % (account, user)
+    headers = {'X-Auth-User-Key': password}
+    if options.admin:
+        headers['X-Auth-User-Admin'] = 'true'
+    conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl)
+    resp = conn.getresponse()
+    if resp.status == 204:
+        print resp.getheader('x-storage-url')
+    else:
+        print 'Update failed: %s %s' % (resp.status, resp.reason)

bin/swift-auth-create-account

@@ -1,47 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2010 OpenStack, LLC.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from ConfigParser import ConfigParser
-from sys import argv, exit
-
-from swift.common.bufferedhttp import http_connect_raw as http_connect
-
-
-if __name__ == '__main__':
-    f = '/etc/swift/auth-server.conf'
-    if len(argv) == 5:
-        f = argv[4]
-    elif len(argv) != 4:
-        exit('Syntax: %s <new_account> <new_user> <new_password> [conf_file]' %
-             argv[0])
-    new_account = argv[1]
-    new_user = argv[2]
-    new_password = argv[3]
-    c = ConfigParser()
-    if not c.read(f):
-        exit('Unable to read conf file: %s' % f)
-    conf = dict(c.items('app:auth-server'))
-    host = conf.get('bind_ip', '127.0.0.1')
-    port = int(conf.get('bind_port', 11000))
-    ssl = conf.get('cert_file') is not None
-    path = '/account/%s/%s' % (new_account, new_user)
-    conn = http_connect(host, port, 'PUT', path, {'x-auth-key':new_password},
-                        ssl=ssl)
-    resp = conn.getresponse()
-    if resp.status == 204:
-        print resp.getheader('x-storage-url')
-    else:
-        print 'Account creation failed. (%d)' % resp.status

bin/swift-stats-populate

@@ -48,7 +48,8 @@ def put_object(connpool, container, obj, report):
     global retries_done
     try:
         with connpool.item() as conn:
-            conn.put_object(container, obj, obj, metadata={'stats': obj})
+            conn.put_object(container, obj, obj,
+                            headers={'x-object-meta-stats': obj})
             retries_done += conn.attempts - 1
         if report:
             report(True)

bin/swift-stats-report

@@ -57,7 +57,8 @@ def get_error_log(prefix):
 def audit(coropool, connpool, account, container_ring, object_ring, options):
     begun = time()
     with connpool.item() as conn:
-        estimated_items = [conn.head_account()[0]]
+        estimated_items = \
+            [int(conn.head_account()['x-account-container-count'])]
     items_completed = [0]
     retries_done = [0]
     containers_missing_replicas = {}
@@ -85,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
                 retries_done[0] += attempts - 1
                 found = True
                 if not estimated_objects:
-                    estimated_objects = info[0]
+                    estimated_objects = int(info['x-container-object-count'])
             except ClientException, err:
                 if err.http_status not in (404, 507):
                     error_log('Giving up on /%s/%s/%s: %s' % (part, account,
@@ -130,7 +131,8 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
     cmarker = ''
     while True:
         with connpool.item() as conn:
-            containers = [c['name'] for c in conn.get_account(marker=cmarker)]
+            containers = \
+                [c['name'] for c in conn.get_account(marker=cmarker)[1]]
         if not containers:
             break
         cmarker = containers[-1]
@@ -142,7 +144,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options):
             while True:
                 with connpool.item() as conn:
                     objects = [o['name'] for o in
-                               conn.get_container(container, marker=omarker)]
+                        conn.get_container(container, marker=omarker)[1]]
                 if not objects:
                     break
                 omarker = objects[-1]
@@ -183,7 +185,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
     with connpool.item() as conn:
         containers = [c['name'] for c in
                       conn.get_account(prefix='stats_container_dispersion_',
-                                       full_listing=True)]
+                                       full_listing=True)[1]]
     containers_listed = len(containers)
     if not containers_listed:
         print >>stderr, 'No containers to query. Has stats-populate been run?'
@@ -262,7 +264,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring, options):
     with connpool.item() as conn:
         try:
             objects = [o['name'] for o in conn.get_container(container,
-                       prefix='stats_object_dispersion_', full_listing=True)]
+                prefix='stats_object_dispersion_', full_listing=True)[1]]
         except ClientException, err:
             if err.http_status != 404:
                 raise
@@ -384,7 +386,7 @@ def container_head_report(coropool, connpool, options):
     with connpool.item() as conn:
         containers = [c['name'] for c in
                       conn.get_account(prefix='stats_container_put_',
-                      full_listing=True)]
+                      full_listing=True)[1]]
     count = len(containers)
     def head(container):
         with connpool.item() as conn:
@@ -425,7 +427,7 @@ def container_get_report(coropool, connpool, options):
     with connpool.item() as conn:
         containers = [c['name'] for c in
                       conn.get_account(prefix='stats_container_put_',
-                      full_listing=True)]
+                      full_listing=True)[1]]
     count = len(containers)
     def get(container):
         with connpool.item() as conn:
@@ -463,7 +465,8 @@ def container_standard_listing_report(coropool, connpool, options):
         print 'Listing big_container',
     with connpool.item() as conn:
         try:
-            value = len(conn.get_container('big_container', full_listing=True))
+            value = \
+                len(conn.get_container('big_container', full_listing=True)[1])
         except ClientException, err:
             if err.http_status != 404:
                 raise
@@ -486,7 +489,7 @@ def container_prefix_listing_report(coropool, connpool, options):
         try:
             for x in xrange(256):
                 value += len(conn.get_container('big_container',
-                                prefix=('%02x' % x), full_listing=True))
+                                prefix=('%02x' % x), full_listing=True)[1])
         except ClientException, err:
             if err.http_status != 404:
                 raise
@@ -511,7 +514,7 @@ def container_prefix_delimiter_listing_report(coropool, connpool, options):
             try:
                 with connpool.item() as conn:
                     listing = conn.get_container('big_container',
-                                marker=marker, prefix=prefix, delimiter='/')
+                                marker=marker, prefix=prefix, delimiter='/')[1]
             except ClientException, err:
                 if err.http_status != 404:
                     raise
@@ -552,7 +555,7 @@ def container_delete_report(coropool, connpool, options):
     with connpool.item() as conn:
         containers = [c['name'] for c in
                       conn.get_account(prefix='stats_container_put_',
-                      full_listing=True)]
+                      full_listing=True)[1]]
     count = len(containers)
     def delete(container):
         with connpool.item() as conn:
@@ -630,7 +633,7 @@ def object_head_report(coropool, connpool, options):
     next_report = [time() + 2]
     with connpool.item() as conn:
         objects = [o['name'] for o in conn.get_container('stats_object_put',
-                   prefix='stats_object_put_', full_listing=True)]
+                   prefix='stats_object_put_', full_listing=True)[1]]
     count = len(objects)
     def head(obj):
         with connpool.item() as conn:
@@ -670,7 +673,7 @@ def object_get_report(coropool, connpool, options):
     next_report = [time() + 2]
     with connpool.item() as conn:
         objects = [o['name'] for o in conn.get_container('stats_object_put',
-                   prefix='stats_object_put_', full_listing=True)]
+                   prefix='stats_object_put_', full_listing=True)[1]]
     count = len(objects)
     def get(obj):
         with connpool.item() as conn:
@@ -710,7 +713,7 @@ def object_delete_report(coropool, connpool, options):
     next_report = [time() + 2]
     with connpool.item() as conn:
         objects = [o['name'] for o in conn.get_container('stats_object_put',
-                   prefix='stats_object_put_', full_listing=True)]
+                   prefix='stats_object_put_', full_listing=True)[1]]
     count = len(objects)
     def delete(obj):
         with connpool.item() as conn:
@@ -791,7 +794,7 @@ Usage: %prog [options] [conf_file]
     else:
         options.retries = int(conf.get('retries', 5))
     if not options.csv_output:
-        csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
+        options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv')
 
     coropool = GreenPool(size=concurrency)
 
@@ -908,8 +911,8 @@ Usage: %prog [options] [conf_file]
 
     if options.csv_output != 'None':
         try:
-            if not os.path.exists(csv_output):
+            if not os.path.exists(options.csv_output):
-                f = open(csv_output, 'wb')
+                f = open(options.csv_output, 'wb')
                 f.write('Timestamp,'
                         'Container Dispersion Report Time,'
                         'Container Dispersion Report Value,'
@@ -936,7 +939,7 @@ Usage: %prog [options] [conf_file]
                         'Object DELETE Report Success Rate\r\n')
                 csv = csv.writer(f)
             else:
-                csv = csv.writer(open(csv_output, 'ab'))
+                csv = csv.writer(open(options.csv_output, 'ab'))
             csv.writerow(report)
         except Exception, err:
             print >>stderr, 'Could not write CSV report:', err

doc/source/auth.rst

@@ -4,7 +4,7 @@
 Developer's Authorization
 *************************
 
-.. _auth-server:
+.. _auth_server:
 
 Auth Server
 ===========

doc/source/development_auth.rst

@@ -0,0 +1,484 @@
+==========================
+Auth Server and Middleware
+==========================
+
+--------------------------------------------
+Creating Your Own Auth Server and Middleware
+--------------------------------------------
+
+The included swift/common/middleware/auth.py is a good minimal example of how
+to create auth middleware. The main points are that the auth middleware can
+reject requests up front, before they ever get to the Swift Proxy application,
+and afterwards when the proxy issues callbacks to verify authorization.
+
+It's generally good to separate the authentication and authorization
+procedures. Authentication verifies that a request actually comes from who it
+says it does. Authorization verifies the 'who' has access to the resource(s)
+the request wants.
+
+Authentication is performed on the request before it ever gets to the Swift
+Proxy application. The identity information is gleaned from the request,
+validated in some way, and the validation information is added to the WSGI
+environment as needed by the future authorization procedure. What exactly is
+added to the WSGI environment is solely dependent on what the installed
+authorization procedures need; the Swift Proxy application itself needs no
+specific information, it just passes it along. Convention has
+environ['REMOTE_USER'] set to the authenticated user string but often more
+information is needed than just that.
+
+The included DevAuth will set the REMOTE_USER to a comma separated list of
+groups the user belongs to. The first group will be the "user's group", a group
+that only the user belongs to. The second group will be the "account's group",
+a group that includes all users for that auth account (different than the
+storage account). The third group is optional and is the storage account
+string. If the user does not have admin access to the account, the third group
+will be omitted.
+
+It is highly recommended that authentication server implementers prefix their
+tokens and Swift storage accounts they create with a configurable reseller
+prefix (`AUTH_` by default with the included DevAuth). This prefix will allow
+deconflicting with other authentication servers that might be using the same
+Swift cluster. Otherwise, the Swift cluster will have to try all the resellers
+until one validates a token or all fail.
+
+A restriction with group names is that no group name should begin with a period
+'.' as that is reserved for internal Swift use (such as the .r for referrer
+designations as you'll see later).
+
+Example Authentication with DevAuth:
+
+    * Token AUTH_tkabcd is given to the DevAuth middleware in a request's
+      X-Auth-Token header.
+    * The DevAuth middleware makes a validate token AUTH_tkabcd call to the
+      external DevAuth server.
+    * The external DevAuth server validates the token AUTH_tkabcd and discovers
+      it matches the "tester" user within the "test" account for the storage
+      account "AUTH_storage_xyz".
+    * The external DevAuth server responds with "X-Auth-Groups:
+      test:tester,test,AUTH_storage_xyz"
+    * Now this user will have full access (via authorization procedures later)
+      to the AUTH_storage_xyz Swift storage account and access to other storage
+      accounts with the same `AUTH_` reseller prefix and has an ACL specifying
+      at least one of those three groups returned.
+
+Authorization is performed through callbacks by the Swift Proxy server to the
+WSGI environment's swift.authorize value, if one is set. The swift.authorize
+value should simply be a function that takes a webob.Request as an argument and
+returns None if access is granted or returns a callable(environ,
+start_response) if access is denied. This callable is a standard WSGI callable.
+Generally, you should return 403 Forbidden for requests by an authenticated
+user and 401 Unauthorized for an unauthenticated request. For example, here's
+an authorize function that only allows GETs (in this case you'd probably return
+405 Method Not Allowed, but ignore that for the moment).::
+
+    from webob import HTTPForbidden, HTTPUnauthorized
+
+
+    def authorize(req):
+        if req.method == 'GET':
+            return None
+        if req.remote_user:
+            return HTTPForbidden(request=req)
+        else:
+            return HTTPUnauthorized(request=req)
+
+Adding the swift.authorize callback is often done by the authentication
+middleware as authentication and authorization are often paired together. But,
+you could create separate authorization middleware that simply sets the
+callback before passing on the request. To continue our example above::
+
+    from webob import HTTPForbidden, HTTPUnauthorized
+
+
+    class Authorization(object):
+
+        def __init__(self, app, conf):
+            self.app = app
+            self.conf = conf
+
+        def __call__(self, environ, start_response):
+            environ['swift.authorize'] = self.authorize
+            return self.app(environ, start_response)
+
+        def authorize(self, req):
+            if req.method == 'GET':
+                return None
+            if req.remote_user:
+                return HTTPForbidden(request=req)
+            else:
+                return HTTPUnauthorized(request=req)
+
+
+    def filter_factory(global_conf, **local_conf):
+        conf = global_conf.copy()
+        conf.update(local_conf)
+        def auth_filter(app):
+            return Authorization(app, conf)
+        return auth_filter
+
+The Swift Proxy server will call swift.authorize after some initial work, but
+before truly trying to process the request. Positive authorization at this
+point will cause the request to be fully processed immediately. A denial at
+this point will immediately send the denial response for most operations.
+
+But for some operations that might be approved with more information, the
+additional information will be gathered and added to the WSGI environment and
+then swift.authorize will be called once more. These are called delay_denial
+requests and currently include container read requests and object read and
+write requests. For these requests, the read or write access control string
+(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl'
+attribute in the webob.Request passed to swift.authorize.
+
+The delay_denial procedures allow skipping possibly expensive access control
+string retrievals for requests that can be approved without that information,
+such as administrator or account owner requests.
+
+To further our example, we now will approve all requests that have the access
+control string set to same value as the authenticated user string. Note that
+you probably wouldn't do this exactly as the access control string represents a
+list rather than a single user, but it'll suffice for this example::
+
+    from webob import HTTPForbidden, HTTPUnauthorized
+
+
+    class Authorization(object):
+
+        def __init__(self, app, conf):
+            self.app = app
+            self.conf = conf
+
+        def __call__(self, environ, start_response):
+            environ['swift.authorize'] = self.authorize
+            return self.app(environ, start_response)
+
+        def authorize(self, req):
+            # Allow anyone to perform GET requests
+            if req.method == 'GET':
+                return None
+            # Allow any request where the acl equals the authenticated user
+            if getattr(req, 'acl', None) == req.remote_user:
+                return None
+            if req.remote_user:
+                return HTTPForbidden(request=req)
+            else:
+                return HTTPUnauthorized(request=req)
+
+
+    def filter_factory(global_conf, **local_conf):
+        conf = global_conf.copy()
+        conf.update(local_conf)
+        def auth_filter(app):
+            return Authorization(app, conf)
+        return auth_filter
+
+The access control string has a standard format included with Swift, though
+this can be overridden if desired. The standard format can be parsed with
+swift.common.middleware.acl.parse_acl which converts the string into two arrays
+of strings: (referrers, groups). The referrers allow comparing the request's
+Referer header to control access. The groups allow comparing the
+request.remote_user (or other sources of group information) to control access.
+Checking referrer access can be accomplished by using the
+swift.common.middleware.acl.referrer_allowed function. Checking group access is
+usually a simple string comparison.
+
+Let's continue our example to use parse_acl and referrer_allowed. Now we'll
+only allow GETs after a referrer check and any requests after a group check::
+
+    from swift.common.middleware.acl import parse_acl, referrer_allowed
+    from webob import HTTPForbidden, HTTPUnauthorized
+
+
+    class Authorization(object):
+
+        def __init__(self, app, conf):
+            self.app = app
+            self.conf = conf
+
+        def __call__(self, environ, start_response):
+            environ['swift.authorize'] = self.authorize
+            return self.app(environ, start_response)
+
+        def authorize(self, req):
+            if hasattr(req, 'acl'):
+                referrers, groups = parse_acl(req.acl)
+                if req.method == 'GET' and referrer_allowed(req, referrers):
+                    return None
+                if req.remote_user and groups and req.remote_user in groups:
+                    return None
+            if req.remote_user:
+                return HTTPForbidden(request=req)
+            else:
+                return HTTPUnauthorized(request=req)
+
+
+    def filter_factory(global_conf, **local_conf):
+        conf = global_conf.copy()
+        conf.update(local_conf)
+        def auth_filter(app):
+            return Authorization(app, conf)
+        return auth_filter
+
+The access control strings are set with PUTs and POSTs to containers with the
+X-Container-Read and X-Container-Write headers. Swift allows these strings to
+be set to any value, though it's very useful to validate the strings meet the
+desired format and return a useful error to the user if they don't.
+
+To support this validation, the Swift Proxy application will call the WSGI
+environment's swift.clean_acl callback whenever one of these headers is to be
+written. The callback should take a header name and value as its arguments. It
+should return the cleaned value to save if valid or raise a ValueError with a
+reasonable error message if not.
+
+There is an included swift.common.middleware.acl.clean_acl that validates the
+standard Swift format. Let's improve our example by making use of that::
+
+    from swift.common.middleware.acl import \
+        clean_acl, parse_acl, referrer_allowed
+    from webob import HTTPForbidden, HTTPUnauthorized
+
+
+    class Authorization(object):
+
+        def __init__(self, app, conf):
+            self.app = app
+            self.conf = conf
+
+        def __call__(self, environ, start_response):
+            environ['swift.authorize'] = self.authorize
+            environ['swift.clean_acl'] = clean_acl
+            return self.app(environ, start_response)
+
+        def authorize(self, req):
+            if hasattr(req, 'acl'):
+                referrers, groups = parse_acl(req.acl)
+                if req.method == 'GET' and referrer_allowed(req, referrers):
+                    return None
+                if req.remote_user and groups and req.remote_user in groups:
+                    return None
+            if req.remote_user:
+                return HTTPForbidden(request=req)
+            else:
+                return HTTPUnauthorized(request=req)
+
+
+    def filter_factory(global_conf, **local_conf):
+        conf = global_conf.copy()
+        conf.update(local_conf)
+        def auth_filter(app):
+            return Authorization(app, conf)
+        return auth_filter
+
+Now, if you want to override the format for access control strings you'll have
+to provide your own clean_acl function and you'll have to do your own parsing
+and authorization checking for that format. It's highly recommended you use the
+standard format simply to support the widest range of external tools, but
+sometimes that's less important than meeting certain ACL requirements.
+
+
+----------------------------
+Integrating With repoze.what
+----------------------------
+
+Here's an example of integration with repoze.what, though honestly it just does
+what the default swift/common/middleware/auth.py does in a slightly different
+way. I'm no repoze.what expert by any stretch; this is just included here to
+hopefully give folks a start on their own code if they want to use
+repoze.what::
+
+    from time import time
+
+    from eventlet.timeout import Timeout
+    from repoze.what.adapters import BaseSourceAdapter
+    from repoze.what.middleware import setup_auth
+    from repoze.what.predicates import in_any_group, NotAuthorizedError
+    from swift.common.bufferedhttp import http_connect_raw as http_connect
+    from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
+    from swift.common.utils import cache_from_env, split_path
+    from webob.exc import HTTPForbidden, HTTPUnauthorized
+
+
+    class DevAuthorization(object):
+
+        def __init__(self, app, conf):
+            self.app = app
+            self.conf = conf
+
+        def __call__(self, environ, start_response):
+            environ['swift.authorize'] = self.authorize
+            environ['swift.clean_acl'] = clean_acl
+            return self.app(environ, start_response)
+
+        def authorize(self, req):
+            version, account, container, obj = split_path(req.path, 1, 4, True)
+            if not account:
+                return self.denied_response(req)
+            referrers, groups = parse_acl(getattr(req, 'acl', None))
+            if referrer_allowed(req, referrers):
+                return None
+            try:
+                in_any_group(account, *groups).check_authorization(req.environ)
+            except NotAuthorizedError:
+                return self.denied_response(req)
+            return None
+
+        def denied_response(self, req):
+            if req.remote_user:
+                return HTTPForbidden(request=req)
+            else:
+                return HTTPUnauthorized(request=req)
+
+
+    class DevIdentifier(object):
+
+        def __init__(self, conf):
+            self.conf = conf
+
+        def identify(self, env):
+            return {'token':
+                    env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))}
+
+        def remember(self, env, identity):
+            return []
+
+        def forget(self, env, identity):
+            return []
+
+
+    class DevAuthenticator(object):
+
+        def __init__(self, conf):
+            self.conf = conf
+            self.auth_host = conf.get('ip', '127.0.0.1')
+            self.auth_port = int(conf.get('port', 11000))
+            self.ssl = \
+                conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes')
+            self.timeout = int(conf.get('node_timeout', 10))
+
+        def authenticate(self, env, identity):
+            token = identity.get('token')
+            if not token:
+                return None
+            memcache_client = cache_from_env(env)
+            key = 'devauth/%s' % token
+            cached_auth_data = memcache_client.get(key)
+            if cached_auth_data:
+                start, expiration, user = cached_auth_data
+                if time() - start <= expiration:
+                    return user
+            with Timeout(self.timeout):
+                conn = http_connect(self.auth_host, self.auth_port, 'GET',
+                                    '/token/%s' % token, ssl=self.ssl)
+                resp = conn.getresponse()
+                resp.read()
+                conn.close()
+            if resp.status == 204:
+                expiration = float(resp.getheader('x-auth-ttl'))
+                user = resp.getheader('x-auth-user')
+                memcache_client.set(key, (time(), expiration, user),
+                                    timeout=expiration)
+                return user
+            return None
+
+
+    class DevChallenger(object):
+
+        def __init__(self, conf):
+            self.conf = conf
+
+        def challenge(self, env, status, app_headers, forget_headers):
+            def no_challenge(env, start_response):
+                start_response(str(status), [])
+                return []
+            return no_challenge
+
+
+    class DevGroupSourceAdapter(BaseSourceAdapter):
+
+        def __init__(self, *args, **kwargs):
+            super(DevGroupSourceAdapter, self).__init__(*args, **kwargs)
+            self.sections = {}
+
+        def _get_all_sections(self):
+            return self.sections
+
+        def _get_section_items(self, section):
+            return self.sections[section]
+
+        def _find_sections(self, credentials):
+            return credentials['repoze.what.userid'].split(',')
+
+        def _include_items(self, section, items):
+            self.sections[section] |= items
+
+        def _exclude_items(self, section, items):
+            for item in items:
+                self.sections[section].remove(item)
+
+        def _item_is_included(self, section, item):
+            return item in self.sections[section]
+
+        def _create_section(self, section):
+            self.sections[section] = set()
+
+        def _edit_section(self, section, new_section):
+            self.sections[new_section] = self.sections[section]
+            del self.sections[section]
+
+        def _delete_section(self, section):
+            del self.sections[section]
+
+        def _section_exists(self, section):
+            return self.sections.has_key(section)
+
+
+    class DevPermissionSourceAdapter(BaseSourceAdapter):
+
+        def __init__(self, *args, **kwargs):
+            super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs)
+            self.sections = {}
+
+        def _get_all_sections(self):
+            return self.sections
+
+        def _get_section_items(self, section):
+            return self.sections[section]
+
+        def _find_sections(self, group_name):
+            return set([n for (n, p) in self.sections.items()
+                        if group_name in p])
+
+        def _include_items(self, section, items):
+            self.sections[section] |= items
+
+        def _exclude_items(self, section, items):
+            for item in items:
+                self.sections[section].remove(item)
+
+        def _item_is_included(self, section, item):
+            return item in self.sections[section]
+
+        def _create_section(self, section):
+            self.sections[section] = set()
+
+        def _edit_section(self, section, new_section):
+            self.sections[new_section] = self.sections[section]
+            del self.sections[section]
+
+        def _delete_section(self, section):
+            del self.sections[section]
+
+        def _section_exists(self, section):
+            return self.sections.has_key(section)
+
+
+    def filter_factory(global_conf, **local_conf):
+        conf = global_conf.copy()
+        conf.update(local_conf)
+        def auth_filter(app):
+            return setup_auth(DevAuthorization(app, conf),
+                group_adapters={'all_groups': DevGroupSourceAdapter()},
+                permission_adapters={'all_perms': DevPermissionSourceAdapter()},
+                identifiers=[('devauth', DevIdentifier(conf))],
+                authenticators=[('devauth', DevAuthenticator(conf))],
+                challengers=[('devauth', DevChallenger(conf))])
+        return auth_filter

doc/source/development_saio.rst

@@ -526,25 +526,17 @@ good idea what to do on other environments.
   #. `remakerings`
   #. `cd ~/swift/trunk; ./.unittests`
   #. `startmain` (The ``Unable to increase file descriptor limit.  Running as non-root?`` warnings are expected and ok.)
-  #. `swift-auth-create-account test tester testing`
+  #. `swift-auth-add-user --admin test tester testing`
   #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0``
   #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>``
   #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat`
-  #. Create `/etc/swift/func_test.conf`::
+  #. `swift-auth-add-user --admin test2 tester2 testing2`
-
+  #. `swift-auth-add-user test tester3 testing3`
-        auth_host = 127.0.0.1
+  #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf`
-        auth_port = 11000
+  #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete
-        auth_ssl = no
+     everything in the configured accounts.)
-
+  #. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your
-        account = test
+     environment as they call `resetswift` for each test.)
-        username = tester
-        password = testing
-
-        collate = C
-
-  #. `cd ~/swift/trunk; ./.functests`
-  #. `cd ~/swift/trunk; ./.probetests` (Note for future reference: probe tests
-     will reset your environment)
 
 If you plan to work on documentation (and who doesn't?!):
 

doc/source/howto_cyberduck.rst

@@ -107,9 +107,9 @@ Installing Swift For Use With Cyberduck
         cert_file = /etc/swift/cert.crt
         key_file = /etc/swift/cert.key
 
-#.  Use swift-auth-create-account to create a new account::
+#.  Use swift-auth-add-user to create a new account and admin user::
 
-        ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-create-account a3 b3 c3
+        ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-add-user --admin a3 b3 c3
         https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1/06228ccf-6d0a-4395-889e-e971e8de8781
 
     .. note::

doc/source/index.rst

@@ -32,6 +32,7 @@ Development:
 
     development_guidelines
     development_saio
+    development_auth
 
 Deployment:
 

doc/source/misc.rst

@@ -42,6 +42,15 @@ Auth
     :members:
     :show-inheritance:
 
+.. _acls:
+
+ACLs
+====
+
+.. automodule:: swift.common.middleware.acl
+    :members:
+    :show-inheritance:
+
 .. _wsgi:
 
 WSGI

doc/source/overview_auth.rst

@@ -6,9 +6,9 @@ The Auth System
 Developer Auth
 --------------
 
-The auth system for Swift is based on the auth system from the existing
+The auth system for Swift is loosely based on the auth system from the existing
-Rackspace architecture -- actually from a few existing auth systems --
+Rackspace architecture -- actually from a few existing auth systems -- and is
-and is therefore a bit disjointed. The distilled points about it are:
+therefore a bit disjointed. The distilled points about it are:
 
 * The authentication/authorization part is outside Swift itself
 * The user of Swift passes in an auth token with each request
@@ -23,13 +23,16 @@ of something unique, some use "something else" but the salient point is that
 the token is a string which can be sent as-is back to the auth system for
 validation.
 
-An auth call is given the auth token and the Swift account hash. For a valid
+Swift will make calls to the external auth system, giving the auth token to be
-token, the auth system responds with a session TTL and overall expiration in
+validated. For a valid token, the auth system responds with an overall
-seconds from now. Swift does not honor the session TTL but will cache the
+expiration in seconds from now. Swift will cache the token up to the expiration
-token up to the expiration time. Tokens can be purged through a call to the
+time. The included devauth also has the concept of admin and non-admin users
-auth system.
+within an account. Admin users can do anything within the account. Non-admin
+users can only perform operations per container based on the container's
+X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see
+:mod:`swift.common.middleware.acl`
 
-The user starts a session by sending a ReST request to that auth system
+The user starts a session by sending a ReST request to the external auth system
 to receive the auth token and a URL to the Swift system.
 
 --------------
@@ -40,8 +43,10 @@ Auth is written as wsgi middleware, so implementing your own auth is as easy
 as writing new wsgi middleware, and plugging it in to the proxy server.
 
 The current middleware is implemented in the DevAuthMiddleware class in
-swift/common/auth.py, and should be a good starting place for implemeting
+swift/common/middleware/auth.py, and should be a good starting place for
-your own auth.
+implementing your own auth.
+
+Also, see :doc:`development_auth`.
 
 ------------------
 History and Future

etc/proxy-server.conf-sample

@@ -37,6 +37,12 @@ use = egg:swift#proxy
 
 [filter:auth]
 use = egg:swift#auth
+# The reseller prefix will verify a token begins with this prefix before even
+# attempting to validate it with the external authentication server. Also, with
+# authorization, only Swift storage accounts with this prefix will be
+# authorized by this middleware. Useful if multiple auth systems are in use for
+# one Swift cluster.
+# reseller_prefix = AUTH
 # ip = 127.0.0.1
 # port = 11000
 # ssl = false

setup.py

@@ -61,7 +61,7 @@ setup(
         'bin/st', 'bin/swift-account-auditor',
         'bin/swift-account-audit', 'bin/swift-account-reaper',
         'bin/swift-account-replicator', 'bin/swift-account-server',
-        'bin/swift-auth-create-account',
+        'bin/swift-auth-add-user',
         'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server',
         'bin/swift-container-auditor',
         'bin/swift-container-replicator',

swift/account/reaper.py

@@ -310,7 +310,7 @@ class AccountReaper(Daemon):
             try:
                 objects = direct_get_container(node, part, account, container,
                         marker=marker, conn_timeout=self.conn_timeout,
-                        response_timeout=self.node_timeout)
+                        response_timeout=self.node_timeout)[1]
                 self.stats_return_codes[2] = \
                     self.stats_return_codes.get(2, 0) + 1
             except ClientException, err:

swift/auth/server.py

@@ -22,6 +22,7 @@ from time import gmtime, strftime, time
 from urllib import unquote, quote
 from uuid import uuid4
 
+import sqlite3
 from webob import Request, Response
 from webob.exc import HTTPBadRequest, HTTPNoContent, HTTPUnauthorized, \
                       HTTPServiceUnavailable, HTTPNotFound
@@ -58,17 +59,18 @@ class AuthController(object):
     * The user makes a ReST call to the Swift cluster using the url given with
       the token as the X-Auth-Token header.
     * The Swift cluster makes an ReST call to the auth server to validate the
-      token for the given account hash, caching the result for future requests
+      token, caching the result for future requests up to the expiration the
-      up to the expiration the auth server returns.
+      auth server returns.
-    * The auth server validates the token / account hash given and returns the
+    * The auth server validates the token given and returns the expiration for
-      expiration for the token.
+      the token.
     * The Swift cluster completes the user's request.
 
-    Another use case is creating a new account:
+    Another use case is creating a new user:
 
-    * The developer makes a ReST call to create a new account.
+    * The developer makes a ReST call to create a new user.
-    * The auth server makes ReST calls to the Swift cluster's account servers
+    * If the account for the user does not yet exist, the auth server makes
-      to create a new account on its end.
+      ReST calls to the Swift cluster's account servers to create a new account
+      on its end.
     * The auth server records the information in its database.
 
     A last use case is recreating existing accounts; this is really only useful
@@ -92,6 +94,9 @@ class AuthController(object):
     def __init__(self, conf, ring=None):
         self.logger = get_logger(conf)
         self.swift_dir = conf.get('swift_dir', '/etc/swift')
+        self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
+        if self.reseller_prefix and self.reseller_prefix[-1] != '_':
+            self.reseller_prefix += '_'
         self.default_cluster_url = \
             conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1')
         self.token_life = int(conf.get('token_life', 86400))
@@ -103,17 +108,33 @@ class AuthController(object):
                 Ring(os.path.join(self.swift_dir, 'account.ring.gz'))
         self.db_file = os.path.join(self.swift_dir, 'auth.db')
         self.conn = get_db_connection(self.db_file, okay_to_create=True)
+        try:
+            self.conn.execute('SELECT admin FROM account LIMIT 1')
+        except sqlite3.OperationalError, err:
+            if str(err) == 'no such column: admin':
+                self.conn.execute("ALTER TABLE account ADD COLUMN admin TEXT")
+                self.conn.execute("UPDATE account SET admin = 't'")
         self.conn.execute('''CREATE TABLE IF NOT EXISTS account (
                                 account TEXT, url TEXT, cfaccount TEXT,
-                                user TEXT, password TEXT)''')
+                                user TEXT, password TEXT, admin TEXT)''')
         self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account
                              ON account (account)''')
+        try:
+            self.conn.execute('SELECT user FROM token LIMIT 1')
+        except sqlite3.OperationalError, err:
+            if str(err) == 'no such column: user':
+                self.conn.execute('DROP INDEX IF EXISTS ix_token_created')
+                self.conn.execute('DROP INDEX IF EXISTS ix_token_cfaccount')
+                self.conn.execute('DROP TABLE IF EXISTS token')
         self.conn.execute('''CREATE TABLE IF NOT EXISTS token (
-                                cfaccount TEXT, token TEXT, created FLOAT)''')
+                                token TEXT, created FLOAT,
-        self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount
+                                account TEXT, user TEXT, cfaccount TEXT)''')
-                             ON token (cfaccount)''')
+        self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token
+                             ON token (token)''')
         self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created
                              ON token (created)''')
+        self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account
+                             ON token (account)''')
         self.conn.commit()
 
     def add_storage_account(self, account_name=''):
@@ -129,7 +150,7 @@ class AuthController(object):
         begin = time()
         orig_account_name = account_name
         if not account_name:
-            account_name = str(uuid4())
+            account_name = '%s%s' % (self.reseller_prefix, uuid4().hex)
         partition, nodes = self.account_ring.get_nodes(account_name)
         headers = {'X-Timestamp': normalize_timestamp(time()),
                    'x-cf-trans-id': 'tx' + str(uuid4())}
@@ -202,77 +223,99 @@ class AuthController(object):
                          (time() - self.token_life,))
             conn.commit()
 
-    def validate_token(self, token, account_hash):
+    def validate_token(self, token):
         """
         Tests if the given token is a valid token
 
         :param token: The token to validate
-        :param account_hash: The account hash the token is being used with
+        :returns: (TTL, account, user, cfaccount) if valid, False otherwise.
-        :returns: TTL if valid, False otherwise
+                  cfaccount will be None for users without admin access.
         """
         begin = time()
         self.purge_old_tokens()
         rv = False
         with self.get_conn() as conn:
             row = conn.execute('''
-                SELECT created FROM token
+                SELECT created, account, user, cfaccount FROM token
-                WHERE cfaccount = ? AND token = ?''',
+                WHERE token = ?''',
-                (account_hash, token)).fetchone()
+                (token,)).fetchone()
             if row is not None:
                 created = row[0]
                 if time() - created >= self.token_life:
                     conn.execute('''
-                        DELETE FROM token
+                        DELETE FROM token WHERE token = ?''', (token,))
-                        WHERE cfaccount = ? AND token = ?''',
-                        (account_hash, token))
                     conn.commit()
                 else:
-                    rv = self.token_life - (time() - created)
+                    rv = (self.token_life - (time() - created), row[1], row[2],
-        self.logger.info('validate_token(%s, %s, _, _) = %s [%.02f]' %
+                          row[3])
-                         (repr(token), repr(account_hash), repr(rv),
+        self.logger.info('validate_token(%s, _, _) = %s [%.02f]' %
-                          time() - begin))
+                         (repr(token), repr(rv), time() - begin))
         return rv
 
-    def create_account(self, new_account, new_user, new_password):
+    def create_user(self, account, user, password, admin=False):
         """
-        Handles the create_account call for developers, used to request
+        Handles the create_user call for developers, used to request a user be
-        an account be created both on a Swift cluster and in the auth server
+        added in the auth server database. If the account does not yet exist,
-        database.
+        it will be created on the Swift cluster and the details recorded in the
+        auth server database.
 
-        This will make ReST requests to the Swift cluster's account servers
+        The url for the storage account is constructed now and stored
-        to have an account created on its side. The resulting account hash
+        separately to support changing the configuration file's
-        along with the URL to use to access the account, the account name, the
+        default_cluster_url for directing new accounts to a different Swift
-        user name, and the password is recorded in the auth server's database.
+        cluster while still supporting old accounts going to the Swift clusters
-        The url is constructed now and stored separately to support changing
+        they were created on.
-        the configuration file's default_cluster_url for directing new accounts
-        to a different Swift cluster while still supporting old accounts going
-        to the Swift clusters they were created on.
 
-        :param new_account: The name for the new account
+        Currently, updating a user's information (password, admin access) must
-        :param new_user: The name for the new user
+        be done by directly updating the sqlite database.
-        :param new_password: The password for the new account
 
-        :returns: False if the create fails, storage url if successful
+        :param account: The name for the new account
+        :param user: The name for the new user
+        :param password: The password for the new account
+        :param admin: If true, the user will be granted full access to the
+                      account; otherwise, another user will have to add the
+                      user to the ACLs for containers to grant access.
+
+        :returns: False if the create fails, 'already exists' if the user
+                  already exists, or storage url if successful
         """
         begin = time()
-        if not all((new_account, new_user, new_password)):
+        if not all((account, user, password)):
             return False
+        with self.get_conn() as conn:
+            row = conn.execute(
+                'SELECT url FROM account WHERE account = ? AND user = ?',
+                (account, user)).fetchone()
+            if row:
+                self.logger.info(
+                    'ALREADY EXISTS create_user(%s, %s, _, %s) [%.02f]' %
+                    (repr(account), repr(user), repr(admin),
+                     time() - begin))
+                return 'already exists'
+            row = conn.execute(
+                'SELECT url, cfaccount FROM account WHERE account = ?',
+                (account,)).fetchone()
+            if row:
+                url = row[0]
+                account_hash = row[1]
+            else:
                 account_hash = self.add_storage_account()
                 if not account_hash:
                     self.logger.info(
-                'FAILED create_account(%s, %s, _,) [%.02f]' %
+                        'FAILED create_user(%s, %s, _, %s) [%.02f]' %
-                (repr(new_account), repr(new_user), time() - begin))
+                        (repr(account), repr(user), repr(admin),
+                         time() - begin))
                     return False
                 url = self.default_cluster_url.rstrip('/') + '/' + account_hash
-        with self.get_conn() as conn:
             conn.execute('''INSERT INTO account
-                (account, url, cfaccount, user, password)
+                (account, url, cfaccount, user, password, admin)
-                VALUES (?, ?, ?, ?, ?)''',
+                VALUES (?, ?, ?, ?, ?, ?)''',
-                (new_account, url, account_hash, new_user, new_password))
+                (account, url, account_hash, user, password,
+                 admin and 't' or ''))
             conn.commit()
         self.logger.info(
-            'SUCCESS create_account(%s, %s, _) = %s [%.02f]' %
+            'SUCCESS create_user(%s, %s, _, %s) = %s [%.02f]' %
-            (repr(new_account), repr(new_user), repr(url), time() - begin))
+            (repr(account), repr(user), repr(admin), repr(url),
+             time() - begin))
         return url
 
     def recreate_accounts(self):
@@ -285,8 +328,8 @@ class AuthController(object):
         """
         begin = time()
         with self.get_conn() as conn:
-            account_hashes = [r[0] for r in
+            account_hashes = [r[0] for r in conn.execute(
-                conn.execute('SELECT cfaccount FROM account').fetchall()]
+                'SELECT distinct(cfaccount) FROM account').fetchall()]
         failures = []
         for i, account_hash in enumerate(account_hashes):
             if not self.add_storage_account(account_hash):
@@ -298,36 +341,47 @@ class AuthController(object):
 
     def handle_token(self, request):
         """
-        Hanles ReST request from Swift to validate tokens
+        Handles ReST requests from Swift to validate tokens
 
         Valid URL paths:
-            * GET /token/<account-hash>/<token>
+            * GET /token/<token>
 
-        If the HTTP equest returns with a 204, then the token is valid,
+        If the HTTP request returns with a 204, then the token is valid, the
-        and the TTL of the token will be available in the X-Auth-Ttl header.
+        TTL of the token will be available in the X-Auth-Ttl header, and a
+        comma separated list of the "groups" the user belongs to will be in the
+        X-Auth-Groups header.
 
         :param request: webob.Request object
         """
         try:
-            _, account_hash, token = split_path(request.path, minsegs=3)
+            _, token = split_path(request.path, minsegs=2)
         except ValueError:
             return HTTPBadRequest()
-        ttl = self.validate_token(token, account_hash)
+        # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise
-        if not ttl:
+        validation = self.validate_token(token)
+        if not validation:
             return HTTPNotFound()
-        return HTTPNoContent(headers={'x-auth-ttl': ttl})
+        groups = ['%s:%s' % (validation[1], validation[2]), validation[1]]
+        if validation[3]: # admin access to a cfaccount
+            groups.append(validation[3])
+        return HTTPNoContent(headers={'X-Auth-TTL': validation[0],
+                                      'X-Auth-Groups': ','.join(groups)})
 
-    def handle_account_create(self, request):
+    def handle_add_user(self, request):
         """
-        Handles Rest requests from developers to have an account created.
+        Handles Rest requests from developers to have a user added. If the
+        account specified doesn't exist, it will also be added. Currently,
+        updating a user's information (password, admin access) must be done by
+        directly updating the sqlite database.
 
         Valid URL paths:
             * PUT /account/<account-name>/<user-name> - create the account
 
         Valid headers:
-            * X-Auth-Key: <password> (Only required when creating an account)
+            * X-Auth-User-Key: <password>
+            * X-Auth-User-Admin: <true|false>
 
-        If the HTTP request returns with a 204, then the account was created,
+        If the HTTP request returns with a 204, then the user was added,
         and the storage url will be available in the X-Storage-Url header.
 
         :param request: webob.Request object
@@ -336,10 +390,13 @@ class AuthController(object):
             _, account_name, user_name = split_path(request.path, minsegs=3)
         except ValueError:
             return HTTPBadRequest()
-        if 'X-Auth-Key' not in request.headers:
+        if 'X-Auth-User-Key' not in request.headers:
-            return HTTPBadRequest('X-Auth-Key is required')
+            return HTTPBadRequest('X-Auth-User-Key is required')
-        password = request.headers['x-auth-key']
+        password = request.headers['x-auth-user-key']
-        storage_url = self.create_account(account_name, user_name, password)
+        storage_url = self.create_user(account_name, user_name, password,
+                        request.headers.get('x-auth-user-admin') == 'true')
+        if storage_url == 'already exists':
+            return HTTPBadRequest(storage_url)
         if not storage_url:
             return HTTPServiceUnavailable()
         return HTTPNoContent(headers={'x-storage-url': storage_url})
@@ -414,23 +471,26 @@ class AuthController(object):
         self.purge_old_tokens()
         with self.get_conn() as conn:
             row = conn.execute('''
-                SELECT cfaccount, url FROM account
+                SELECT cfaccount, url, admin FROM account
                 WHERE account = ? AND user = ? AND password = ?''',
                 (account, user, password)).fetchone()
             if row is None:
                 return HTTPUnauthorized()
             cfaccount = row[0]
             url = row[1]
-            row = conn.execute('SELECT token FROM token WHERE cfaccount = ?',
+            admin = row[2] == 't'
-                               (cfaccount,)).fetchone()
+            row = conn.execute('''
+                SELECT token FROM token WHERE account = ? AND user = ?''',
+                (account, user)).fetchone()
             if row:
                 token = row[0]
             else:
-                token = 'tk' + str(uuid4())
+                token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
                 conn.execute('''
-                    INSERT INTO token (cfaccount, token, created)
+                    INSERT INTO token
-                    VALUES (?, ?, ?)''',
+                    (token, created, account, user, cfaccount)
-                    (cfaccount, token, time()))
+                    VALUES (?, ?, ?, ?, ?)''',
+                    (token, time(), account, user, admin and cfaccount or ''))
                 conn.commit()
             return HTTPNoContent(headers={'x-auth-token': token,
                                           'x-storage-token': token,
@@ -457,7 +517,7 @@ class AuthController(object):
             elif req.method == 'GET' and req.path.startswith('/token/'):
                 handler = self.handle_token
             elif req.method == 'PUT' and req.path.startswith('/account/'):
-                handler = self.handle_account_create
+                handler = self.handle_add_user
             elif req.method == 'POST' and \
                     req.path == '/recreate_accounts':
                 handler = self.handle_account_recreate

swift/common/client.py

@@ -194,18 +194,21 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
                       conn object)
     :param full_listing: if True, return a full listing, else returns a max
                          of 10000 listings
-    :returns: a list of accounts
+    :returns: a tuple of (response headers, a list of containers) The response
+              headers will be a dict and all header names will be lowercase. 
     :raises ClientException: HTTP GET request failed
     """
     if not http_conn:
         http_conn = http_connection(url)
     if full_listing:
-        rv = []
+        rv = get_account(url, token, marker, limit, prefix, http_conn)
-        listing = get_account(url, token, marker, limit, prefix, http_conn)
+        listing = rv[1]
         while listing:
-            rv.extend(listing)
             marker = listing[-1]['name']
-            listing = get_account(url, token, marker, limit, prefix, http_conn)
+            listing = \
+                get_account(url, token, marker, limit, prefix, http_conn)[1]
+            if listing:
+                rv.extend(listing)
         return rv
     parsed, conn = http_conn
     qs = 'format=json'
@@ -218,6 +221,9 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
     conn.request('GET', '%s?%s' % (parsed.path, qs), '',
                  {'X-Auth-Token': token})
     resp = conn.getresponse()
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
     if resp.status < 200 or resp.status >= 300:
         resp.read()
         raise ClientException('Account GET failed', http_scheme=parsed.scheme,
@@ -226,8 +232,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None,
                 http_reason=resp.reason)
     if resp.status == 204:
         resp.read()
-        return []
+        return resp_headers, []
-    return json_loads(resp.read())
+    return resp_headers, json_loads(resp.read())
 
 
 def head_account(url, token, http_conn=None):
@@ -238,7 +244,8 @@ def head_account(url, token, http_conn=None):
     :param token: auth token
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
-    :returns: a tuple of (container count, object count, bytes used)
+    :returns: a dict containing the response's headers (all header names will
+              be lowercase)
     :raises ClientException: HTTP HEAD request failed
     """
     if http_conn:
@@ -253,9 +260,36 @@ def head_account(url, token, http_conn=None):
                 http_host=conn.host, http_port=conn.port,
                 http_path=parsed.path, http_status=resp.status,
                 http_reason=resp.reason)
-    return int(resp.getheader('x-account-container-count', 0)), \
+    resp_headers = {}
-           int(resp.getheader('x-account-object-count', 0)), \
+    for header, value in resp.getheaders():
-           int(resp.getheader('x-account-bytes-used', 0))
+        resp_headers[header.lower()] = value
+    return resp_headers
+
+
+def post_account(url, token, headers, http_conn=None):
+    """
+    Update an account's metadata.
+
+    :param url: storage URL
+    :param token: auth token
+    :param headers: additional headers to include in the request
+    :param http_conn: HTTP connection object (If None, it will create the
+                      conn object)
+    :raises ClientException: HTTP POST request failed
+    """
+    if http_conn:
+        parsed, conn = http_conn
+    else:
+        parsed, conn = http_connection(url)
+    headers['X-Auth-Token'] = token
+    conn.request('POST', parsed.path, '', headers)
+    resp = conn.getresponse()
+    resp.read()
+    if resp.status < 200 or resp.status >= 300:
+        raise ClientException('Account POST failed',
+                http_scheme=parsed.scheme, http_host=conn.host,
+                http_port=conn.port, http_path=path, http_status=resp.status,
+                http_reason=resp.reason)
 
 
 def get_container(url, token, container, marker=None, limit=None,
@@ -275,23 +309,25 @@ def get_container(url, token, container, marker=None, limit=None,
                       conn object)
     :param full_listing: if True, return a full listing, else returns a max
                          of 10000 listings
-    :returns: a list of objects
+    :returns: a tuple of (response headers, a list of objects) The response
+              headers will be a dict and all header names will be lowercase. 
     :raises ClientException: HTTP GET request failed
     """
     if not http_conn:
         http_conn = http_connection(url)
     if full_listing:
-        rv = []
+        rv = get_container(url, token, container, marker, limit, prefix,
-        listing = get_container(url, token, container, marker, limit, prefix,
                            delimiter, http_conn)
+        listing = rv[1]
         while listing:
-            rv.extend(listing)
             if not delimiter:
                 marker = listing[-1]['name']
             else:
                 marker = listing[-1].get('name', listing[-1].get('subdir'))
             listing = get_container(url, token, container, marker, limit,
-                                    prefix, delimiter, http_conn)
+                                    prefix, delimiter, http_conn)[1]
+            if listing:
+                rv[1].extend(listing)
         return rv
     parsed, conn = http_conn
     path = '%s/%s' % (parsed.path, quote(container))
@@ -312,10 +348,13 @@ def get_container(url, token, container, marker=None, limit=None,
                 http_scheme=parsed.scheme, http_host=conn.host,
                 http_port=conn.port, http_path=path, http_query=qs,
                 http_status=resp.status, http_reason=resp.reason)
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
     if resp.status == 204:
         resp.read()
-        return []
+        return resp_headers, []
-    return json_loads(resp.read())
+    return resp_headers, json_loads(resp.read())
 
 
 def head_container(url, token, container, http_conn=None):
@@ -327,7 +366,8 @@ def head_container(url, token, container, http_conn=None):
     :param container: container name to get stats for
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
-    :returns: a tuple of (object count, bytes used)
+    :returns: a dict containing the response's headers (all header names will
+              be lowercase)
     :raises ClientException: HTTP HEAD request failed
     """
     if http_conn:
@@ -343,17 +383,20 @@ def head_container(url, token, container, http_conn=None):
                 http_scheme=parsed.scheme, http_host=conn.host,
                 http_port=conn.port, http_path=path, http_status=resp.status,
                 http_reason=resp.reason)
-    return int(resp.getheader('x-container-object-count', 0)), \
+    resp_headers = {}
-           int(resp.getheader('x-container-bytes-used', 0))
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
+    return resp_headers
 
 
-def put_container(url, token, container, http_conn=None):
+def put_container(url, token, container, headers=None, http_conn=None):
     """
     Create a container
 
     :param url: storage URL
     :param token: auth token
     :param container: container name to create
+    :param headers: additional headers to include in the request
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
     :raises ClientException: HTTP PUT request failed
@@ -363,7 +406,10 @@ def put_container(url, token, container, http_conn=None):
     else:
         parsed, conn = http_connection(url)
     path = '%s/%s' % (parsed.path, quote(container))
-    conn.request('PUT', path, '', {'X-Auth-Token': token})
+    if not headers:
+        headers = {}
+    headers['X-Auth-Token'] = token
+    conn.request('PUT', path, '', headers)
     resp = conn.getresponse()
     resp.read()
     if resp.status < 200 or resp.status >= 300:
@@ -373,6 +419,34 @@ def put_container(url, token, container, http_conn=None):
                 http_reason=resp.reason)
 
 
+def post_container(url, token, container, headers, http_conn=None):
+    """
+    Update a container's metadata.
+
+    :param url: storage URL
+    :param token: auth token
+    :param container: container name to update
+    :param headers: additional headers to include in the request
+    :param http_conn: HTTP connection object (If None, it will create the
+                      conn object)
+    :raises ClientException: HTTP POST request failed
+    """
+    if http_conn:
+        parsed, conn = http_conn
+    else:
+        parsed, conn = http_connection(url)
+    path = '%s/%s' % (parsed.path, quote(container))
+    headers['X-Auth-Token'] = token
+    conn.request('POST', path, '', headers)
+    resp = conn.getresponse()
+    resp.read()
+    if resp.status < 200 or resp.status >= 300:
+        raise ClientException('Container POST failed',
+                http_scheme=parsed.scheme, http_host=conn.host,
+                http_port=conn.port, http_path=path, http_status=resp.status,
+                http_reason=resp.reason)
+
+
 def delete_container(url, token, container, http_conn=None):
     """
     Delete a container
@@ -410,8 +484,12 @@ def get_object(url, token, container, name, http_conn=None,
     :param name: object name to get
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
-    :param resp_chunk_size: if defined, chunk size of data to read
+    :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If
-    :returns: a list of objects
+                            you specify a resp_chunk_size you must fully read
+                            the object's contents before making another
+                            request.
+    :returns: a tuple of (response headers, the object's contents) The response
+              headers will be a dict and all header names will be lowercase. 
     :raises ClientException: HTTP GET request failed
     """
     if http_conn:
@@ -426,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None,
         raise ClientException('Object GET failed', http_scheme=parsed.scheme,
                 http_host=conn.host, http_port=conn.port, http_path=path,
                 http_status=resp.status, http_reason=resp.reason)
-    metadata = {}
-    for key, value in resp.getheaders():
-        if key.lower().startswith('x-object-meta-'):
-            metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
     if resp_chunk_size:
 
         def _object_body():
@@ -440,12 +514,10 @@ def get_object(url, token, container, name, http_conn=None,
         object_body = _object_body()
     else:
         object_body = resp.read()
-    return resp.getheader('content-type'), \
+    resp_headers = {}
-           int(resp.getheader('content-length', 0)), \
+    for header, value in resp.getheaders():
-           resp.getheader('last-modified'), \
+        resp_headers[header.lower()] = value
-           resp.getheader('etag').strip('"'), \
+    return resp_headers, object_body
-           metadata, \
-           object_body
 
 
 def head_object(url, token, container, name, http_conn=None):
@@ -458,8 +530,8 @@ def head_object(url, token, container, name, http_conn=None):
     :param name: object name to get info for
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
-    :returns: a tuple of (content type, content length, last modfied, etag,
+    :returns: a dict containing the response's headers (all header names will
-              dictionary of metadata)
+              be lowercase)
     :raises ClientException: HTTP HEAD request failed
     """
     if http_conn:
@@ -474,20 +546,15 @@ def head_object(url, token, container, name, http_conn=None):
         raise ClientException('Object HEAD failed', http_scheme=parsed.scheme,
                 http_host=conn.host, http_port=conn.port, http_path=path,
                 http_status=resp.status, http_reason=resp.reason)
-    metadata = {}
+    resp_headers = {}
-    for key, value in resp.getheaders():
+    for header, value in resp.getheaders():
-        if key.lower().startswith('x-object-meta-'):
+        resp_headers[header.lower()] = value
-            metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
+    return resp_headers
-    return resp.getheader('content-type'), \
-           int(resp.getheader('content-length', 0)), \
-           resp.getheader('last-modified'), \
-           resp.getheader('etag').strip('"'), \
-           metadata
 
 
-def put_object(url, token, container, name, contents, metadata={},
+def put_object(url, token, container, name, contents, content_length=None,
-               content_length=None, etag=None, chunk_size=65536,
+               etag=None, chunk_size=65536, content_type=None, headers=None,
-               content_type=None, http_conn=None):
+               http_conn=None):
     """
     Put an object
 
@@ -495,12 +562,12 @@ def put_object(url, token, container, name, contents, metadata={},
     :param token: auth token
     :param container: container name that the object is in
     :param name: object name to put
-    :param contents: file like object to read object data from
+    :param contents: a string or a file like object to read object data from
-    :param metadata: dictionary of object metadata
     :param content_length: value to send as content-length header
     :param etag: etag of contents
     :param chunk_size: chunk size of data to write
     :param content_type: value to send as content-type header
+    :param headers: additional headers to include in the request
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
     :returns: etag from server response
@@ -511,9 +578,9 @@ def put_object(url, token, container, name, contents, metadata={},
     else:
         parsed, conn = http_connection(url)
     path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
-    headers = {'X-Auth-Token': token}
+    if not headers:
-    for key, value in metadata.iteritems():
+        headers = {}
-        headers['X-Object-Meta-%s' % quote(key)] = quote(value)
+    headers['X-Auth-Token'] = token
     if etag:
         headers['ETag'] = etag.strip('"')
     if content_length is not None:
@@ -549,15 +616,15 @@ def put_object(url, token, container, name, contents, metadata={},
     return resp.getheader('etag').strip('"')
 
 
-def post_object(url, token, container, name, metadata, http_conn=None):
+def post_object(url, token, container, name, headers, http_conn=None):
     """
-    Change object metadata
+    Update object metadata
 
     :param url: storage URL
     :param token: auth token
     :param container: container name that the object is in
-    :param name: object name to change
+    :param name: name of the object to update
-    :param metadata: dictionary of object metadata
+    :param headers: additional headers to include in the request
     :param http_conn: HTTP connection object (If None, it will create the
                       conn object)
     :raises ClientException: HTTP POST request failed
@@ -567,9 +634,7 @@ def post_object(url, token, container, name, metadata, http_conn=None):
     else:
         parsed, conn = http_connection(url)
     path = '%s/%s/%s' % (parsed.path, quote(container), quote(name))
-    headers = {'X-Auth-Token': token}
+    headers['X-Auth-Token'] = token
-    for key, value in metadata.iteritems():
-        headers['X-Object-Meta-%s' % quote(key)] = quote(value)
     conn.request('POST', path, '', headers)
     resp = conn.getresponse()
     resp.read()
@@ -682,6 +747,10 @@ class Connection(object):
         return self._retry(get_account, marker=marker, limit=limit,
                            prefix=prefix, full_listing=full_listing)
 
+    def post_account(self, headers):
+        """Wrapper for :func:`post_account`"""
+        return self._retry(post_account, headers)
+
     def head_container(self, container):
         """Wrapper for :func:`head_container`"""
         return self._retry(head_container, container)
@@ -696,9 +765,13 @@ class Connection(object):
                            limit=limit, prefix=prefix, delimiter=delimiter,
                            full_listing=full_listing)
 
-    def put_container(self, container):
+    def put_container(self, container, headers=None):
         """Wrapper for :func:`put_container`"""
-        return self._retry(put_container, container)
+        return self._retry(put_container, container, headers=headers)
+
+    def post_container(self, container, headers):
+        """Wrapper for :func:`post_container`"""
+        return self._retry(post_container, container, headers)
 
     def delete_container(self, container):
         """Wrapper for :func:`delete_container`"""
@@ -713,17 +786,17 @@ class Connection(object):
         return self._retry(get_object, container, obj,
                            resp_chunk_size=resp_chunk_size)
 
-    def put_object(self, container, obj, contents, metadata={},
+    def put_object(self, container, obj, contents, content_length=None,
-                   content_length=None, etag=None, chunk_size=65536,
+                   etag=None, chunk_size=65536, content_type=None,
-                   content_type=None):
+                   headers=None):
         """Wrapper for :func:`put_object`"""
         return self._retry(put_object, container, obj, contents,
-            metadata=metadata, content_length=content_length, etag=etag,
+            content_length=content_length, etag=etag, chunk_size=chunk_size,
-            chunk_size=chunk_size, content_type=content_type)
+            content_type=content_type, headers=headers)
 
-    def post_object(self, container, obj, metadata):
+    def post_object(self, container, obj, headers):
         """Wrapper for :func:`post_object`"""
-        return self._retry(post_object, container, obj, metadata)
+        return self._retry(post_object, container, obj, headers)
 
     def delete_object(self, container, obj):
         """Wrapper for :func:`delete_object`"""

swift/common/direct_client.py

@@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
     :param container: container name
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
-    :returns: tuple of (object count, bytes used)
+    :returns: a dict containing the response's headers (all header names will
+              be lowercase)
     """
     path = '/%s/%s' % (account, container)
     with Timeout(conn_timeout):
@@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
                 http_host=node['ip'], http_port=node['port'],
                 http_device=node['device'], http_status=resp.status,
                 http_reason=resp.reason)
-    return int(resp.getheader('x-container-object-count')), \
+    resp_headers = {}
-           int(resp.getheader('x-container-bytes-used'))
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
+    return resp_headers
 
 
 def direct_get_container(node, part, account, container, marker=None,
@@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None,
     :param delimeter: delimeter for the query
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
-    :returns: list of objects
+    :returns: a tuple of (response headers, a list of objects) The response
+              headers will be a dict and all header names will be lowercase. 
     """
     path = '/%s/%s' % (account, container)
     qs = 'format=json'
@@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None,
             http_host=node['ip'], http_port=node['port'],
             http_device=node['device'], http_status=resp.status,
             http_reason=resp.reason)
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
     if resp.status == 204:
         resp.read()
-        return []
+        return resp_headers, []
-    return json_loads(resp.read())
+    return resp_headers, json_loads(resp.read())
 
 
 def direct_delete_container(node, part, account, container, conn_timeout=5,
@@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
                 'DELETE', path, headers)
     with Timeout(response_timeout):
         resp = conn.getresponse()
+        resp.read()
     if resp.status < 200 or resp.status >= 300:
         raise ClientException(
                 'Container server %s:%s direct DELETE %s gave status %s' %
@@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5,
                 http_host=node['ip'], http_port=node['port'],
                 http_device=node['device'], http_status=resp.status,
                 http_reason=resp.reason)
-    return resp
 
 
 def direct_head_object(node, part, account, container, obj, conn_timeout=5,
@@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
     :param obj: object name
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
-    :returns: tuple of (content-type, object size, last modified timestamp,
+    :returns: a dict containing the response's headers (all header names will
-              etag, metadata dictionary)
+              be lowercase)
     """
     path = '/%s/%s/%s' % (account, container, obj)
     with Timeout(conn_timeout):
@@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5,
                 http_host=node['ip'], http_port=node['port'],
                 http_device=node['device'], http_status=resp.status,
                 http_reason=resp.reason)
-    metadata = {}
+    resp_headers = {}
-    for key, value in resp.getheaders():
+    for header, value in resp.getheaders():
-        if key.lower().startswith('x-object-meta-'):
+        resp_headers[header.lower()] = value
-            metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
+    return resp_headers
-    return resp.getheader('content-type'), \
-           int(resp.getheader('content-length')), \
-           resp.getheader('last-modified'), \
-           resp.getheader('etag').strip('"'), \
-           metadata
 
 
 def direct_get_object(node, part, account, container, obj, conn_timeout=5,
-        response_timeout=15):
+                      response_timeout=15, resp_chunk_size=None):
     """
     Get object directly from the object server.
 
@@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
     :param obj: object name
     :param conn_timeout: timeout in seconds for establishing the connection
     :param response_timeout: timeout in seconds for getting the response
-    :returns: object
+    :param resp_chunk_size: if defined, chunk size of data to read.
+    :returns: a tuple of (response headers, the object's contents) The response
+              headers will be a dict and all header names will be lowercase. 
     """
     path = '/%s/%s/%s' % (account, container, obj)
     with Timeout(conn_timeout):
@@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
     with Timeout(response_timeout):
         resp = conn.getresponse()
     if resp.status < 200 or resp.status >= 300:
+        resp.read()
         raise ClientException(
                 'Object server %s:%s direct GET %s gave status %s' %
                 (node['ip'], node['port'],
@@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
                 http_host=node['ip'], http_port=node['port'],
                 http_device=node['device'], http_status=resp.status,
                 http_reason=resp.reason)
-    metadata = {}
+    if resp_chunk_size:
-    for key, value in resp.getheaders():
+
-        if key.lower().startswith('x-object-meta-'):
+        def _object_body():
-            metadata[unquote(key[len('x-object-meta-'):])] = unquote(value)
+            buf = resp.read(resp_chunk_size)
-    return (resp.getheader('content-type'),
+            while buf:
-           int(resp.getheader('content-length')),
+                yield buf
-           resp.getheader('last-modified'),
+                buf = resp.read(resp_chunk_size)
-           resp.getheader('etag').strip('"'),
+        object_body = _object_body()
-           metadata,
+    else:
-           resp.read())
+        object_body = resp.read()
+    resp_headers = {}
+    for header, value in resp.getheaders():
+        resp_headers[header.lower()] = value
+    return resp_headers, object_body
 
 
 def direct_delete_object(node, part, account, container, obj,
@@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj,
                 'DELETE', path, headers)
     with Timeout(response_timeout):
         resp = conn.getresponse()
+        resp.read()
     if resp.status < 200 or resp.status >= 300:
         raise ClientException(
                 'Object server %s:%s direct DELETE %s gave status %s' %
@@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj,
                 http_host=node['ip'], http_port=node['port'],
                 http_device=node['device'], http_status=resp.status,
                 http_reason=resp.reason)
-    return resp
 
 
 def retry(func, *args, **kwargs):

swift/common/middleware/acl.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from urlparse import urlparse
+
+
+def clean_acl(name, value):
+    """
+    Returns a cleaned ACL header value, validating that it meets the formatting
+    requirements for standard Swift ACL strings.
+
+    The ACL format is::
+
+        [item[,item...]]
+
+    Each item can be a group name to give access to or a referrer designation
+    to grant or deny based on the HTTP Referer header.
+
+    The referrer designation format is::
+
+        .r:[-]value
+
+    The ``.r`` can also be ``.ref``, ``.referer``, or ``.referrer``; though it
+    will be shortened to just ``.r`` for decreased character count usage.
+
+    The value can be ``*`` to specify any referrer host is allowed access, a
+    specific host name like ``www.example.com``, or if it has a leading period
+    ``.`` or leading ``*.`` it is a domain name specification, like
+    ``.example.com`` or ``*.example.com``. The leading minus sign ``-``
+    indicates referrer hosts that should be denied access.
+
+    Referrer access is applied in the order they are specified. For example,
+    .r:.example.com,.r:-thief.example.com would allow all hosts ending with
+    .example.com except for the specific host thief.example.com.
+
+    Example valid ACLs::
+
+        .r:*
+        .r:*,.r:-.thief.com
+        .r:*,.r:.example.com,.r:-thief.example.com
+        .r:*,.r:-.thief.com,bobs_account,sues_account:sue
+        bobs_account,sues_account:sue
+
+    Example invalid ACLs::
+
+        .r:
+        .r:-
+
+    Also, .r designations aren't allowed in headers whose names include the
+    word 'write'.
+
+    ACLs that are "messy" will be cleaned up. Examples:
+
+    ======================  ======================
+    Original                Cleaned
+    ----------------------  ----------------------
+    ``bob, sue``            ``bob,sue``
+    ``bob , sue``           ``bob,sue``
+    ``bob,,,sue``           ``bob,sue``
+    ``.referrer : *``       ``.r:*``
+    ``.ref:*.example.com``  ``.r:.example.com``
+    ======================  ======================
+
+    :param name: The name of the header being cleaned, such as X-Container-Read
+                 or X-Container-Write.
+    :param value: The value of the header being cleaned.
+    :returns: The value, cleaned of extraneous formatting.
+    :raises ValueError: If the value does not meet the ACL formatting
+                        requirements; the error message will indicate why.
+    """
+    name = name.lower()
+    values = []
+    for raw_value in value.split(','):
+        raw_value = raw_value.strip()
+        if raw_value:
+            if ':' not in raw_value:
+                values.append(raw_value)
+            else:
+                first, second = (v.strip() for v in raw_value.split(':', 1))
+                if not first or first[0] != '.':
+                    values.append(raw_value)
+                elif first in ('.r', '.ref', '.referer', '.referrer'):
+                    if 'write' in name:
+                        raise ValueError('Referrers not allowed in write ACL: '
+                                         '%s' % repr(raw_value))
+                    negate = False
+                    if second and second[0] == '-':
+                        negate = True
+                        second = second[1:].strip()
+                    if second and second != '*' and second[0] == '*':
+                        second = second[1:].strip()
+                    if not second or second == '.':
+                        raise ValueError('No host/domain value after referrer '
+                            'designation in ACL: %s' % repr(raw_value))
+                    values.append('.r:%s%s' % (negate and '-' or '', second))
+                else:
+                    raise ValueError('Unknown designator %s in ACL: %s' %
+                                     (repr(first), repr(raw_value)))
+    return ','.join(values)
+
+
+def parse_acl(acl_string):
+    """
+    Parses a standard Swift ACL string into a referrers list and groups list.
+
+    See :func:`clean_acl` for documentation of the standard Swift ACL format.
+
+    :param acl_string: The standard Swift ACL string to parse.
+    :returns: A tuple of (referrers, groups) where referrers is a list of
+              referrer designations (without the leading .r:) and groups is a
+              list of groups to allow access.
+    """
+    referrers = []
+    groups = []
+    if acl_string:
+        for value in acl_string.split(','):
+            if value.startswith('.r:'):
+                referrers.append(value[len('.r:'):])
+            else:
+                groups.append(value)
+    return referrers, groups
+
+
+def referrer_allowed(referrer, referrer_acl):
+    """
+    Returns True if the referrer should be allowed based on the referrer_acl
+    list (as returned by :func:`parse_acl`).
+
+    See :func:`clean_acl` for documentation of the standard Swift ACL format.
+
+    :param referrer: The value of the HTTP Referer header.
+    :param referrer_acl: The list of referrer designations as returned by
+                         :func:`parse_acl`.
+    :returns: True if the referrer should be allowed; False if not.
+    """
+    allow = False
+    if referrer_acl:
+        rhost = urlparse(referrer or '').hostname or 'unknown'
+        for mhost in referrer_acl:
+            if mhost[0] == '-':
+                mhost = mhost[1:]
+                if mhost == rhost or \
+                       (mhost[0] == '.' and rhost.endswith(mhost)):
+                    allow = False
+            elif mhost == '*' or mhost == rhost or \
+                    (mhost[0] == '.' and rhost.endswith(mhost)):
+                allow = True
+    return allow

swift/common/middleware/auth.py

@@ -13,31 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
+from time import time
 
-from webob.request import Request
-from webob.exc import HTTPUnauthorized, HTTPPreconditionFailed
 from eventlet.timeout import Timeout
+from webob.exc import HTTPForbidden, HTTPUnauthorized
 
-from swift.common.utils import split_path
 from swift.common.bufferedhttp import http_connect_raw as http_connect
-from swift.common.utils import get_logger, cache_from_env
+from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
-from swift.common.memcached import MemcacheRing
+from swift.common.utils import cache_from_env, split_path
 
 
-class DevAuthMiddleware(object):
+class DevAuth(object):
-    """
+    """Auth Middleware that uses the dev auth server."""
-    Auth Middleware that uses the dev auth server
-    """
 
-    def __init__(self, app, conf, memcache_client=None, logger=None):
+    def __init__(self, app, conf):
         self.app = app
-        self.memcache_client = memcache_client
-        if logger is None:
-            self.logger = get_logger(conf)
-        else:
-            self.logger = logger
         self.conf = conf
+        self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip()
+        if self.reseller_prefix and self.reseller_prefix[-1] != '_':
+            self.reseller_prefix += '_'
         self.auth_host = conf.get('ip', '127.0.0.1')
         self.auth_port = int(conf.get('port', 11000))
         self.ssl = \
@@ -45,68 +39,79 @@ class DevAuthMiddleware(object):
         self.timeout = int(conf.get('node_timeout', 10))
 
     def __call__(self, env, start_response):
-        if self.memcache_client is None:
-            self.memcache_client = cache_from_env(env)
-        req = Request(env)
-        if 'x-storage-token' in req.headers and \
-                'x-auth-token' not in req.headers:
-            req.headers['x-auth-token'] = req.headers['x-storage-token']
-        try:
-            version, account, container, obj = split_path(req.path, 1, 4, True)
-        except ValueError, e:
-            version = account = container = obj = None
-        if account is None:
-            return HTTPPreconditionFailed(request=req, body='Bad URL')(
-                env, start_response)
-        if not req.headers.get('x-auth-token'):
-            return HTTPPreconditionFailed(request=req,
-                body='Missing Auth Token')(env, start_response)
-        if not self.auth(account, req.headers['x-auth-token']):
-            return HTTPUnauthorized(request=req)(env, start_response)
-
-        # If we get here, then things should be good.
-        return self.app(env, start_response)
-
-    def auth(self, account, token):
         """
-        Dev authorization implmentation
+        Accepts a standard WSGI application call, authenticating the request
-
+        and installing callback hooks for authorization and ACL header
-        :param account: account name
+        validation. For an authenticated request, REMOTE_USER will be set to a
-        :param token: auth token
+        comma separated list of the user's groups.
-
-        :returns: True if authorization is successful, False otherwise
         """
-        key = 'auth/%s/%s' % (account, token)
+        groups = None
-        now = time.time()
+        token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
-        cached_auth_data = self.memcache_client.get(key)
+        if token and token.startswith(self.reseller_prefix):
+            memcache_client = cache_from_env(env)
+            key = 'devauth/%s' % token
+            cached_auth_data = memcache_client.get(key)
             if cached_auth_data:
-            start, expiration = cached_auth_data
+                start, expiration, groups = cached_auth_data
-            if now - start <= expiration:
+                if time() - start > expiration:
-                return True
+                    groups = None
-        try:
+            if not groups:
                 with Timeout(self.timeout):
                     conn = http_connect(self.auth_host, self.auth_port, 'GET',
-                    '/token/%s/%s' % (account, token), ssl=self.ssl)
+                                        '/token/%s' % token, ssl=self.ssl)
                     resp = conn.getresponse()
                     resp.read()
                     conn.close()
-                if resp.status == 204:
+                if resp.status // 100 != 2:
-                    validated = float(resp.getheader('x-auth-ttl'))
+                    return HTTPUnauthorized()(env, start_response)
+                expiration = float(resp.getheader('x-auth-ttl'))
+                groups = resp.getheader('x-auth-groups')
+                memcache_client.set(key, (time(), expiration, groups),
+                                    timeout=expiration)
+        env['REMOTE_USER'] = groups
+        env['swift.authorize'] = self.authorize
+        env['swift.clean_acl'] = clean_acl
+        # We know the proxy logs the token, so we augment it just a bit to also
+        # log the authenticated user.
+        user = groups and groups.split(',', 1)[0] or ''
+        env['HTTP_X_AUTH_TOKEN'] = '%s,%s' % (user, token)
+        return self.app(env, start_response)
+
+    def authorize(self, req):
+        """
+        Returns None if the request is authorized to continue or a standard
+        WSGI response callable if not.
+        """
+        version, account, container, obj = split_path(req.path, 1, 4, True)
+        if not account or not account.startswith(self.reseller_prefix):
+            return self.denied_response(req)
+        if req.remote_user and account in req.remote_user.split(','):
+            return None
+        referrers, groups = parse_acl(getattr(req, 'acl', None))
+        if referrer_allowed(req.referer, referrers):
+            return None
+        if not req.remote_user:
+            return self.denied_response(req)
+        for user_group in req.remote_user.split(','):
+            if user_group in groups:
+                return None
+        return self.denied_response(req)
+
+    def denied_response(self, req):
+        """
+        Returns a standard WSGI response callable with the status of 403 or 401
+        depending on whether the REMOTE_USER is set or not.
+        """
+        if req.remote_user:
+            return HTTPForbidden(request=req)
         else:
-                    validated = False
+            return HTTPUnauthorized(request=req)
-        except:
+
-            self.logger.exception('ERROR with auth')
-            return False
-        if not validated:
-            return False
-        else:
-            val = (now, validated)
-            self.memcache_client.set(key, val, timeout=validated)
-            return True
 
 def filter_factory(global_conf, **local_conf):
+    """Returns a WSGI filter app for use with paste.deploy."""
     conf = global_conf.copy()
     conf.update(local_conf)
     def auth_filter(app):
-        return DevAuthMiddleware(app, conf)
+        return DevAuth(app, conf)
     return auth_filter

swift/container/server.py

@@ -44,6 +44,9 @@ DATADIR = 'containers'
 class ContainerController(object):
     """WSGI Controller for the container server."""
 
+    # Ensure these are all lowercase
+    save_headers = ['x-container-read', 'x-container-write']
+
     def __init__(self, conf):
         self.logger = get_logger(conf)
         self.root = conf.get('devices', '/srv/node/')
@@ -192,7 +195,8 @@ class ContainerController(object):
             metadata = {}
             metadata.update((key, (value, timestamp))
                 for key, value in req.headers.iteritems()
-                if key.lower().startswith('x-container-meta-'))
+                if key.lower() in self.save_headers or
+                   key.lower().startswith('x-container-meta-'))
             if metadata:
                 broker.update_metadata(metadata)
             resp = self.account_update(req, account, container, broker)
@@ -373,7 +377,8 @@ class ContainerController(object):
         metadata = {}
         metadata.update((key, (value, timestamp))
             for key, value in req.headers.iteritems()
-            if key.lower().startswith('x-container-meta-'))
+            if key.lower() in self.save_headers or
+               key.lower().startswith('x-container-meta-'))
         if metadata:
             broker.update_metadata(metadata)
         return HTTPNoContent(request=req)

swift/proxy/server.py

@@ -17,6 +17,7 @@ from __future__ import with_statement
 import mimetypes
 import os
 import time
+import traceback
 from ConfigParser import ConfigParser
 from urllib import unquote, quote
 import uuid
@@ -73,6 +74,22 @@ def public(func):
     return wrapped
 
 
+def delay_denial(func):
+    """
+    Decorator to declare which methods should have any swift.authorize call
+    delayed. This is so the method can load the Request object up with
+    additional information that may be needed by the authorization system.
+
+    :param func: function to delay authorization on
+    """
+    func.delay_denial = True
+
+    @functools.wraps(func)
+    def wrapped(*a, **kw):
+        return func(*a, **kw)
+    return wrapped
+
+
 class Controller(object):
     """Base WSGI controller class for the proxy"""
 
@@ -206,19 +223,28 @@ class Controller(object):
 
         :param account: account name for the container
         :param container: container name to look up
-        :returns: tuple of (container partition, container nodes) or
+        :returns: tuple of (container partition, container nodes, container
-                           (None, None) if the container does not exist
+                  read acl, container write acl) or (None, None, None, None) if
+                  the container does not exist
         """
         partition, nodes = self.app.container_ring.get_nodes(
                 account, container)
         path = '/%s/%s' % (account, container)
         cache_key = 'container%s' % path
+        # Older memcache values (should be treated as if they aren't there):
         # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
-        if self.app.memcache.get(cache_key) == 200:
+        # Newer memcache values:
-            return partition, nodes
+        # [older status value from above, read acl, write acl]
+        cache_value = self.app.memcache.get(cache_key)
+        if hasattr(cache_value, '__iter__'):
+            status, read_acl, write_acl = cache_value
+            if status == 200:
+                return partition, nodes, read_acl, write_acl
         if not self.account_info(account)[1]:
-            return (None, None)
+            return (None, None, None, None)
         result_code = 0
+        read_acl = None
+        write_acl = None
         attempts_left = self.app.container_ring.replica_count
         headers = {'x-cf-trans-id': self.trans_id}
         for node in self.iter_nodes(partition, nodes, self.app.container_ring):
@@ -233,6 +259,8 @@ class Controller(object):
                     body = resp.read()
                     if 200 <= resp.status <= 299:
                         result_code = 200
+                        read_acl = resp.getheader('x-container-read')
+                        write_acl = resp.getheader('x-container-write')
                         break
                     elif resp.status == 404:
                         result_code = 404 if not result_code else -1
@@ -251,10 +279,11 @@ class Controller(object):
             cache_timeout = self.app.recheck_container_existence
         else:
             cache_timeout = self.app.recheck_container_existence * 0.1
-        self.app.memcache.set(cache_key, result_code, timeout=cache_timeout)
+        self.app.memcache.set(cache_key, (result_code, read_acl, write_acl),
+                              timeout=cache_timeout)
         if result_code == 200:
-            return partition, nodes
+            return partition, nodes, read_acl, write_acl
-        return (None, None)
+        return (None, None, None, None)
 
     def iter_nodes(self, partition, nodes, ring):
         """
@@ -474,6 +503,12 @@ class ObjectController(Controller):
 
     def GETorHEAD(self, req):
         """Handle HTTP GET or HEAD requests."""
+        if 'swift.authorize' in req.environ:
+            req.acl = \
+                self.container_info(self.account_name, self.container_name)[2]
+            aresp = req.environ['swift.authorize'](req)
+            if aresp:
+                return aresp
         partition, nodes = self.app.object_ring.get_nodes(
             self.account_name, self.container_name, self.object_name)
         return self.GETorHEAD_base(req, 'Object', partition,
@@ -481,13 +516,30 @@ class ObjectController(Controller):
                 req.path_info, self.app.object_ring.replica_count)
 
     @public
+    @delay_denial
+    def GET(self, req):
+        """Handler for HTTP GET requests."""
+        return self.GETorHEAD(req)
+
+    @public
+    @delay_denial
+    def HEAD(self, req):
+        """Handler for HTTP HEAD requests."""
+        return self.GETorHEAD(req)
+
+    @public
+    @delay_denial
     def POST(self, req):
         """HTTP POST request handler."""
         error_response = check_metadata(req, 'object')
         if error_response:
             return error_response
-        container_partition, containers = \
+        container_partition, containers, _, req.acl = \
             self.container_info(self.account_name, self.container_name)
+        if 'swift.authorize' in req.environ:
+            aresp = req.environ['swift.authorize'](req)
+            if aresp:
+                return aresp
         if not containers:
             return HTTPNotFound(request=req)
         containers = self.get_update_nodes(container_partition, containers,
@@ -521,10 +573,15 @@ class ObjectController(Controller):
                 bodies, 'Object POST')
 
     @public
+    @delay_denial
     def PUT(self, req):
         """HTTP PUT request handler."""
-        container_partition, containers = \
+        container_partition, containers, _, req.acl = \
             self.container_info(self.account_name, self.container_name)
+        if 'swift.authorize' in req.environ:
+            aresp = req.environ['swift.authorize'](req)
+            if aresp:
+                return aresp
         if not containers:
             return HTTPNotFound(request=req)
         containers = self.get_update_nodes(container_partition, containers,
@@ -701,10 +758,15 @@ class ObjectController(Controller):
         return resp
 
     @public
+    @delay_denial
     def DELETE(self, req):
         """HTTP DELETE request handler."""
-        container_partition, containers = \
+        container_partition, containers, _, req.acl = \
             self.container_info(self.account_name, self.container_name)
+        if 'swift.authorize' in req.environ:
+            aresp = req.environ['swift.authorize'](req)
+            if aresp:
+                return aresp
         if not containers:
             return HTTPNotFound(request=req)
         containers = self.get_update_nodes(container_partition, containers,
@@ -771,11 +833,26 @@ class ObjectController(Controller):
 class ContainerController(Controller):
     """WSGI controller for container requests"""
 
+    # Ensure these are all lowercase
+    pass_through_headers = ['x-container-read', 'x-container-write']
+
     def __init__(self, app, account_name, container_name, **kwargs):
         Controller.__init__(self, app)
         self.account_name = unquote(account_name)
         self.container_name = unquote(container_name)
 
+    def clean_acls(self, req):
+        if 'swift.clean_acl' in req.environ:
+            for header in ('x-container-read', 'x-container-write'):
+                if header in req.headers:
+                    try:
+                        req.headers[header] = \
+                            req.environ['swift.clean_acl'](header,
+                                                           req.headers[header])
+                    except ValueError, err:
+                        return HTTPBadRequest(request=req, body=str(err))
+        return None
+
     def GETorHEAD(self, req):
         """Handler for HTTP GET/HEAD requests."""
         if not self.account_info(self.account_name)[1]:
@@ -784,12 +861,30 @@ class ContainerController(Controller):
                         self.account_name, self.container_name)
         resp = self.GETorHEAD_base(req, 'Container', part, nodes,
                 req.path_info, self.app.container_ring.replica_count)
+        if 'swift.authorize' in req.environ:
+            req.acl = resp.headers.get('x-container-read')
+            aresp = req.environ['swift.authorize'](req)
+            if aresp:
+                return aresp
         return resp
 
+    @public
+    @delay_denial
+    def GET(self, req):
+        """Handler for HTTP GET requests."""
+        return self.GETorHEAD(req)
+
+    @public
+    @delay_denial
+    def HEAD(self, req):
+        """Handler for HTTP HEAD requests."""
+        return self.GETorHEAD(req)
+
     @public
     def PUT(self, req):
         """HTTP PUT request handler."""
-        error_response = check_metadata(req, 'container')
+        error_response = \
+            self.clean_acls(req) or check_metadata(req, 'container')
         if error_response:
             return error_response
         if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
@@ -807,7 +902,8 @@ class ContainerController(Controller):
         headers = {'X-Timestamp': normalize_timestamp(time.time()),
                    'x-cf-trans-id': self.trans_id}
         headers.update(value for value in req.headers.iteritems()
-            if value[0].lower().startswith('x-container-meta-'))
+            if value[0].lower() in self.pass_through_headers or
+               value[0].lower().startswith('x-container-meta-'))
         statuses = []
         reasons = []
         bodies = []
@@ -853,7 +949,8 @@ class ContainerController(Controller):
     @public
     def POST(self, req):
         """HTTP POST request handler."""
-        error_response = check_metadata(req, 'container')
+        error_response = \
+            self.clean_acls(req) or check_metadata(req, 'container')
         if error_response:
             return error_response
         account_partition, accounts = self.account_info(self.account_name)
@@ -864,7 +961,8 @@ class ContainerController(Controller):
         headers = {'X-Timestamp': normalize_timestamp(time.time()),
                    'x-cf-trans-id': self.trans_id}
         headers.update(value for value in req.headers.iteritems()
-            if value[0].lower().startswith('x-container-meta-'))
+            if value[0].lower() in self.pass_through_headers or
+               value[0].lower().startswith('x-container-meta-'))
         statuses = []
         reasons = []
         bodies = []
@@ -1118,7 +1216,8 @@ class BaseApplication(object):
                 self.posthooklogger(env, req)
                 return response
         except:
-            print "EXCEPTION IN __call__: %s" % env
+            print "EXCEPTION IN __call__: %s: %s" % \
+                  (traceback.format_exc(), env)
             start_response('500 Server Error',
                     [('Content-Type', 'text/plain')])
             return ['Internal server error.\n']
@@ -1160,12 +1259,30 @@ class BaseApplication(object):
             controller.trans_id = req.headers.get('x-cf-trans-id', '-')
             try:
                 handler = getattr(controller, req.method)
-                if getattr(handler, 'publicly_accessible'):
+                if not getattr(handler, 'publicly_accessible'):
+                    handler = None
+            except AttributeError:
+                handler = None
+            if not handler:
+                return HTTPMethodNotAllowed(request=req)
             if path_parts['version']:
                 req.path_info_pop()
+            if 'swift.authorize' in req.environ:
+                # We call authorize before the handler, always. If authorized,
+                # we remove the swift.authorize hook so isn't ever called
+                # again. If not authorized, we return the denial unless the
+                # controller's method indicates it'd like to gather more
+                # information and try again later.
+                resp = req.environ['swift.authorize'](req)
+                if not resp:
+                    # No resp means authorized, no delayed recheck required.
+                    del req.environ['swift.authorize']
+                else:
+                    # Response indicates denial, but we might delay the denial
+                    # and recheck later. If not delayed, return the error now.
+                    if not getattr(handler, 'delay_denial', None):
+                        return resp
             return handler(req)
-            except AttributeError:
-                return HTTPMethodNotAllowed(request=req)
         except Exception:
             self.logger.exception('ERROR Unhandled exception in request')
             return HTTPServerError(request=req)
@@ -1187,7 +1304,9 @@ class Application(BaseApplication):
         return req.response
 
     def posthooklogger(self, env, req):
-        response = req.response
+        response = getattr(req, 'response', None)
+        if not response:
+            return
         trans_time = '%.4f' % (time.time() - req.start_time)
         the_request = quote(unquote(req.path))
         if req.query_string:

test/functional/sample.conf

@@ -1,10 +1,20 @@
-# Sample functional test configuration file
+# sample config
 auth_host = 127.0.0.1
-auth_port = 80
+auth_port = 11000
 auth_ssl = no
 
-account  = test_account
+# Primary functional test account (needs admin access to the account)
-username = test_user
+account = test
-password = test_password
+username = tester
+password = testing
+
+# User on a second account (needs admin access to the account)
+account2 = test2
+username2 = tester2
+password2 = testing2
+
+# User on same account as first, but without admin access
+username3 = tester3
+password3 = testing3
 
 collate = C

test/functional/tests.py

@@ -106,9 +106,12 @@ class Base(unittest.TestCase):
         self.assert_(response_body == body,
            'Body returned: %s' % (response_body))
 
-    def assert_status(self, status):
+    def assert_status(self, status_or_statuses):
-        self.assert_(self.env.conn.response.status == status,
+        self.assert_(self.env.conn.response.status == status_or_statuses or
-           'Status returned: %d' % (self.env.conn.response.status))
+                     (hasattr(status_or_statuses, '__iter__') and
+                      self.env.conn.response.status in status_or_statuses),
+           'Status returned: %d Expected: %s' %
+           (self.env.conn.response.status, status_or_statuses))
 
 class Base2(object):
     def setUp(self):
@@ -148,11 +151,11 @@ class TestAccount(Base):
     def testNoAuthToken(self):
         self.assertRaises(ResponseError, self.env.account.info,
             cfg={'no_auth_token':True})
-        self.assert_status(412)
+        self.assert_status([401, 412])
 
         self.assertRaises(ResponseError, self.env.account.containers,
             cfg={'no_auth_token':True})
-        self.assert_status(412)
+        self.assert_status([401, 412])
 
     def testInvalidUTF8Path(self):
         invalid_utf8 = Utils.create_utf8_name()[::-1]
@@ -1123,7 +1126,8 @@ class TestFile(Base):
         self.assert_status(400)
 
         # bad request types
-        for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'):
+        #for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'):
+        for req in ('LICK', 'GETorHEAD_base'):
             self.env.account.conn.make_request(req)
             self.assert_status(405)
 

test/functionalnosetests/swift_testing.py

@@ -10,11 +10,11 @@ from swift.common.client import get_auth, http_connection
 
 
 swift_test_auth = os.environ.get('SWIFT_TEST_AUTH')
-swift_test_user = os.environ.get('SWIFT_TEST_USER')
+swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None]
-swift_test_key = os.environ.get('SWIFT_TEST_KEY')
+swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None]
 
 # If no environment set, fall back to old school conf file
-if not all([swift_test_auth, swift_test_user, swift_test_key]):
+if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]):
     conf = ConfigParser()
     class Sectionizer(object):
         def __init__(self, fp):
@@ -32,16 +32,36 @@ if not all([swift_test_auth, swift_test_user, swift_test_key]):
         if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'):
             swift_test_auth = 'https'
         swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf
-        swift_test_user = '%(account)s:%(username)s' % conf
+        swift_test_user[0] = '%(account)s:%(username)s' % conf
-        swift_test_key = conf['password']
+        swift_test_key[0] = conf['password']
+        try:
+            swift_test_user[1] = '%(account2)s:%(username2)s' % conf
+            swift_test_key[1] = conf['password2']
+        except KeyError, err:
+            pass # old conf, no second account tests can be run
+        try:
+            swift_test_user[2] = '%(account)s:%(username3)s' % conf
+            swift_test_key[2] = conf['password3']
+        except KeyError, err:
+            pass # old conf, no third account tests can be run
     except IOError, err:
         if err.errno != errno.ENOENT:
             raise
 
-skip = not all([swift_test_auth, swift_test_user, swift_test_key])
+skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]])
 if skip:
     print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG'
 
+skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]])
+if not skip and skip2:
+    print >>sys.stderr, \
+          'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
+
+skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]])
+if not skip and skip3:
+    print >>sys.stderr, \
+          'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM'
+
 
 class AuthError(Exception):
     pass
@@ -51,29 +71,44 @@ class InternalServerError(Exception):
     pass
 
 
-url = token = parsed = conn = None
+url = [None, None, None]
+token = [None, None, None]
+parsed = [None, None, None]
+conn  = [None, None, None]
 
 def retry(func, *args, **kwargs):
+    """
+    You can use the kwargs to override the 'retries' (default: 5) and
+    'use_account' (default: 1).
+    """
     global url, token, parsed, conn
     retries = kwargs.get('retries', 5)
+    use_account = 1
+    if 'use_account' in kwargs:
+        use_account = kwargs['use_account']
+        del kwargs['use_account']
+    use_account -= 1
     attempts = 0
     backoff = 1
     while attempts <= retries:
         attempts += 1
         try:
-            if not url or not token:
+            if not url[use_account] or not token[use_account]:
-                url, token = \
+                url[use_account], token[use_account] = \
-                    get_auth(swift_test_auth, swift_test_user, swift_test_key)
+                    get_auth(swift_test_auth, swift_test_user[use_account],
-                parsed = conn = None
+                             swift_test_key[use_account])
-            if not parsed or not conn:
+                parsed[use_account] = conn[use_account] = None
-                parsed, conn = http_connection(url)
+            if not parsed[use_account] or not conn[use_account]:
-            return func(url, token, parsed, conn, *args, **kwargs)
+                parsed[use_account], conn[use_account] = \
+                    http_connection(url[use_account])
+            return func(url[use_account], token[use_account],
+                       parsed[use_account], conn[use_account], *args, **kwargs)
         except (socket.error, HTTPException):
             if attempts > retries:
                 raise
-            parsed = conn = None
+            parsed[use_account] = conn[use_account] = None
         except AuthError, err:
-            url = token = None
+            url[use_account] = token[use_account] = None
             continue
         except InternalServerError, err:
             pass

test/functionalnosetests/test_account.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import unittest
+from nose import SkipTest
 
 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
     MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
@@ -12,7 +13,7 @@ class TestAccount(unittest.TestCase):
 
     def test_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, value):
             conn.request('POST', parsed.path, '',
                 {'X-Auth-Token': token, 'X-Account-Meta-Test': value})
@@ -48,7 +49,7 @@ class TestAccount(unittest.TestCase):
 
     def test_multi_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, name, value):
             conn.request('POST', parsed.path, '',
                          {'X-Auth-Token': token, name: value})
@@ -74,7 +75,7 @@ class TestAccount(unittest.TestCase):
 
     def test_bad_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, extra_headers):
             headers = {'X-Auth-Token': token}
             headers.update(extra_headers)

test/functionalnosetests/test_container.py

@@ -1,19 +1,22 @@
 #!/usr/bin/python
 
+import json
 import unittest
+from nose import SkipTest
 from uuid import uuid4
 
 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
     MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
 
-from swift_testing import check_response, retry, skip
+from swift_testing import check_response, retry, skip, skip2, skip3, \
+                          swift_test_user
 
 
 class TestContainer(unittest.TestCase):
 
     def setUp(self):
         if skip:
-            return
+            raise SkipTest
         self.name = uuid4().hex
         def put(url, token, parsed, conn):
             conn.request('PUT', parsed.path + '/' + self.name, '',
@@ -25,7 +28,27 @@ class TestContainer(unittest.TestCase):
 
     def tearDown(self):
         if skip:
-            return
+            raise SkipTest
+        def get(url, token, parsed, conn):
+            conn.request('GET', parsed.path + '/' + self.name + '?format=json',
+                         '', {'X-Auth-Token': token})
+            return check_response(conn)
+        def delete(url, token, parsed, conn, obj):
+            conn.request('DELETE',
+                         '/'.join([parsed.path, self.name, obj['name']]), '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        while True:
+            resp = retry(get)
+            body = resp.read()
+            self.assert_(resp.status // 100 == 2, resp.status)
+            objs = json.loads(body)
+            if not objs:
+                break
+            for obj in objs:
+                resp = retry(delete, obj)
+                resp.read()
+                self.assertEquals(resp.status, 204)
         def delete(url, token, parsed, conn):
             conn.request('DELETE', parsed.path + '/' + self.name, '',
                          {'X-Auth-Token': token})
@@ -36,7 +59,7 @@ class TestContainer(unittest.TestCase):
 
     def test_multi_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, name, value):
             conn.request('POST', parsed.path + '/' + self.name, '',
                 {'X-Auth-Token': token, name: value})
@@ -63,7 +86,7 @@ class TestContainer(unittest.TestCase):
 
     def test_PUT_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def put(url, token, parsed, conn, name, value):
             conn.request('PUT', parsed.path + '/' + name, '',
                 {'X-Auth-Token': token, 'X-Container-Meta-Test': value})
@@ -110,7 +133,7 @@ class TestContainer(unittest.TestCase):
 
     def test_POST_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, value):
             conn.request('POST', parsed.path + '/' + self.name, '',
                 {'X-Auth-Token': token, 'X-Container-Meta-Test': value})
@@ -145,7 +168,7 @@ class TestContainer(unittest.TestCase):
 
     def test_PUT_bad_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def put(url, token, parsed, conn, name, extra_headers):
             headers = {'X-Auth-Token': token}
             headers.update(extra_headers)
@@ -240,7 +263,7 @@ class TestContainer(unittest.TestCase):
 
     def test_POST_bad_metadata(self):
         if skip:
-            return
+            raise SkipTest
         def post(url, token, parsed, conn, extra_headers):
             headers = {'X-Auth-Token': token}
             headers.update(extra_headers)
@@ -297,6 +320,204 @@ class TestContainer(unittest.TestCase):
         resp.read()
         self.assertEquals(resp.status, 400)
 
+    def test_public_container(self):
+        if skip:
+            raise SkipTest
+        def get(url, token, parsed, conn):
+            conn.request('GET', parsed.path + '/' + self.name)
+            return check_response(conn)
+        try:
+            resp = retry(get)
+            raise Exception('Should not have been able to GET')
+        except Exception, err:
+            self.assert_(str(err).startswith('No result after '), err)
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token,
+                          'X-Container-Read': '.r:*'})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        resp = retry(get)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token, 'X-Container-Read': ''})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        try:
+            resp = retry(get)
+            raise Exception('Should not have been able to GET')
+        except Exception, err:
+            self.assert_(str(err).startswith('No result after '), err)
+
+    def test_cross_account_container(self):
+        if skip or skip2:
+            raise SkipTest
+        # Obtain the first account's string
+        first_account = ['unknown']
+        def get1(url, token, parsed, conn):
+            first_account[0] = parsed.path
+            conn.request('HEAD', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get1)
+        resp.read()
+        # Ensure we can't access the container with the second account
+        def get2(url, token, parsed, conn):
+            conn.request('GET', first_account[0] + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+        # Make the container accessible by the second account
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                {'X-Auth-Token': token, 'X-Container-Read': 'test2',
+                 'X-Container-Write': 'test2'})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can now use the container with the second account
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Make the container private again
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token, 'X-Container-Read': '',
+                          'X-Container-Write': ''})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can't access the container with the second account again
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+
+    def test_cross_account_public_container(self):
+        if skip or skip2:
+            raise SkipTest
+        # Obtain the first account's string
+        first_account = ['unknown']
+        def get1(url, token, parsed, conn):
+            first_account[0] = parsed.path
+            conn.request('HEAD', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get1)
+        resp.read()
+        # Ensure we can't access the container with the second account
+        def get2(url, token, parsed, conn):
+            conn.request('GET', first_account[0] + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+        # Make the container completely public
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token,
+                          'X-Container-Read': '.r:*'})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can now read the container with the second account
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # But we shouldn't be able to write with the second account
+        def put2(url, token, parsed, conn):
+            conn.request('PUT', first_account[0] + '/' + self.name + '/object',
+                         'test object', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(put2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+        # Now make the container also writeable by the second account
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                {'X-Auth-Token': token, 'X-Container-Write': 'test2'})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can still read the container with the second account
+        resp = retry(get2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # And that we can now write with the second account
+        resp = retry(put2, use_account=2)
+        resp.read()
+        self.assertEquals(resp.status, 201)
+
+    def test_nonadmin_user(self):
+        if skip or skip3:
+            raise SkipTest
+        # Obtain the first account's string
+        first_account = ['unknown']
+        def get1(url, token, parsed, conn):
+            first_account[0] = parsed.path
+            conn.request('HEAD', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get1)
+        resp.read()
+        # Ensure we can't access the container with the third account
+        def get3(url, token, parsed, conn):
+            conn.request('GET', first_account[0] + '/' + self.name, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(get3, use_account=3)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+        # Make the container accessible by the third account
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+               {'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can now read the container with the third account
+        resp = retry(get3, use_account=3)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # But we shouldn't be able to write with the third account
+        def put3(url, token, parsed, conn):
+            conn.request('PUT', first_account[0] + '/' + self.name + '/object',
+                         'test object', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(put3, use_account=3)
+        resp.read()
+        self.assertEquals(resp.status, 403)
+        # Now make the container also writeable by the third account
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.name, '',
+                         {'X-Auth-Token': token,
+                          'X-Container-Write': swift_test_user[2]})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # Ensure we can still read the container with the third account
+        resp = retry(get3, use_account=3)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        # And that we can now write with the third account
+        resp = retry(put3, use_account=3)
+        resp.read()
+        self.assertEquals(resp.status, 201)
+
 
 if __name__ == '__main__':
     unittest.main()

test/functionalnosetests/test_object.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+
+import unittest
+from nose import SkipTest
+from uuid import uuid4
+
+from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \
+    MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH
+
+from swift_testing import check_response, retry, skip
+
+
+class TestObject(unittest.TestCase):
+
+    def setUp(self):
+        if skip:
+            raise SkipTest
+        self.container = uuid4().hex
+        def put(url, token, parsed, conn):
+            conn.request('PUT', parsed.path + '/' + self.container, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(put)
+        resp.read()
+        self.assertEquals(resp.status, 201)
+        self.obj = uuid4().hex
+        def put(url, token, parsed, conn):
+            conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container,
+                self.obj), 'test', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(put)
+        resp.read()
+        self.assertEquals(resp.status, 201)
+
+    def tearDown(self):
+        if skip:
+            raise SkipTest
+        def delete(url, token, parsed, conn):
+            conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container,
+                self.obj), '', {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(delete)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        def delete(url, token, parsed, conn):
+            conn.request('DELETE', parsed.path + '/' + self.container, '',
+                         {'X-Auth-Token': token})
+            return check_response(conn)
+        resp = retry(delete)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+
+    def test_public_object(self):
+        if skip:
+            raise SkipTest
+        def get(url, token, parsed, conn):
+            conn.request('GET',
+                         '%s/%s/%s' % (parsed.path, self.container, self.obj))
+            return check_response(conn)
+        try:
+            resp = retry(get)
+            raise Exception('Should not have been able to GET')
+        except Exception, err:
+            self.assert_(str(err).startswith('No result after '))
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.container, '',
+                         {'X-Auth-Token': token,
+                          'X-Container-Read': '.r:*'})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        resp = retry(get)
+        resp.read()
+        self.assertEquals(resp.status, 200)
+        def post(url, token, parsed, conn):
+            conn.request('POST', parsed.path + '/' + self.container, '',
+                         {'X-Auth-Token': token, 'X-Container-Read': ''})
+            return check_response(conn)
+        resp = retry(post)
+        resp.read()
+        self.assertEquals(resp.status, 204)
+        try:
+            resp = retry(get)
+            raise Exception('Should not have been able to GET')
+        except Exception, err:
+            self.assert_(str(err).startswith('No result after '))
+
+
+if __name__ == '__main__':
+    unittest.main()

test/probe/test_account_failures.py

@@ -39,8 +39,10 @@ class TestAccountFailures(unittest.TestCase):
         client.put_container(self.url, self.token, container1)
         container2 = 'container2'
         client.put_container(self.url, self.token, container2)
-        self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
+        headers, containers = client.get_account(self.url, self.token)
-        containers = client.get_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '2')
+        self.assertEquals(headers['x-account-object-count'], '0')
+        self.assertEquals(headers['x-account-bytes-used'], '0')
         found1 = False
         found2 = False
         for c in containers:
@@ -56,8 +58,10 @@ class TestAccountFailures(unittest.TestCase):
         self.assert_(found2)
 
         client.put_object(self.url, self.token, container2, 'object1', '1234')
-        self.assert_(client.head_account(self.url, self.token), (2, 0, 0))
+        headers, containers = client.get_account(self.url, self.token)
-        containers = client.get_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '2')
+        self.assertEquals(headers['x-account-object-count'], '0')
+        self.assertEquals(headers['x-account-bytes-used'], '0')
         found1 = False
         found2 = False
         for c in containers:
@@ -73,8 +77,10 @@ class TestAccountFailures(unittest.TestCase):
         self.assert_(found2)
 
         get_to_final_state()
-        containers = client.get_account(self.url, self.token)
+        headers, containers = client.get_account(self.url, self.token)
-        self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
+        self.assertEquals(headers['x-account-container-count'], '2')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
         found2 = False
         for c in containers:
@@ -94,8 +100,10 @@ class TestAccountFailures(unittest.TestCase):
 
         client.delete_container(self.url, self.token, container1)
         client.put_object(self.url, self.token, container2, 'object2', '12345')
-        self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
+        headers, containers = client.get_account(self.url, self.token)
-        containers = client.get_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
         found2 = False
         for c in containers:
@@ -115,8 +123,10 @@ class TestAccountFailures(unittest.TestCase):
                              'once']))
         for p in ps:
             p.wait()
-        self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
+        headers, containers = client.get_account(self.url, self.token)
-        containers = client.get_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '2')
+        self.assertEquals(headers['x-account-bytes-used'], '9')
         found1 = False
         found2 = False
         for c in containers:
@@ -134,10 +144,12 @@ class TestAccountFailures(unittest.TestCase):
                    '/etc/swift/account-server/%d.conf' %
                     ((anodes[0]['port'] - 6002) / 10)]).pid
         sleep(2)
-        # This is the earlier object count and bytes because the first node
+        # This is the earlier counts and bytes because the first node doesn't
-        # doesn't have the newest udpates yet.
+        # have the newest udpates yet.
-        self.assert_(client.head_account(self.url, self.token), (2, 1, 4))
+        headers, containers = client.get_account(self.url, self.token)
-        containers = client.get_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '2')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
         found2 = False
         for c in containers:
@@ -155,8 +167,10 @@ class TestAccountFailures(unittest.TestCase):
         self.assert_(found2)
 
         get_to_final_state()
-        containers = client.get_account(self.url, self.token)
+        headers, containers = client.get_account(self.url, self.token)
-        self.assert_(client.head_account(self.url, self.token), (2, 2, 9))
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '2')
+        self.assertEquals(headers['x-account-bytes-used'], '9')
         found1 = False
         found2 = False
         for c in containers:

test/probe/test_container_failures.py

@@ -40,23 +40,23 @@ class TestContainerFailures(unittest.TestCase):
         container = 'container-%s' % uuid4()
         client.put_container(self.url, self.token, container)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
 
         object1 = 'object1'
         client.put_object(self.url, self.token, container, object1, 'test')
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         cpart, cnodes = self.container_ring.get_nodes(self.account, container)
         kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM)
 
         client.delete_object(self.url, self.token, container, object1)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         self.pids[self.port2server[cnodes[0]['port']]] = \
             Popen(['swift-container-server',
@@ -64,11 +64,11 @@ class TestContainerFailures(unittest.TestCase):
                     ((cnodes[0]['port'] - 6001) / 10)]).pid
         sleep(2)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # This okay because the first node hasn't got the update that the
         # object was deleted yet.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -87,7 +87,7 @@ class TestContainerFailures(unittest.TestCase):
         # account server, this'll pass, otherwise the first account server will
         # serve the listing and not have the container.
         # self.assert_(container in [c['name'] for c in
-        #              client.get_account(self.url, self.token)])
+        #              client.get_account(self.url, self.token)[1]])
 
         object2 = 'object2'
         # This will work because at least one (in this case, just one) account
@@ -95,44 +95,44 @@ class TestContainerFailures(unittest.TestCase):
         client.put_object(self.url, self.token, container, object2, 'test')
         # First node still doesn't know object1 was deleted yet; this is okay.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         get_to_final_state()
         # Our container delete never "finalized" because we started using it
         # before the delete settled.
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # And, so our object2 should still exist and object1's delete should
         # have finalized.
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
     def test_second_node_fail(self):
         container = 'container-%s' % uuid4()
         client.put_container(self.url, self.token, container)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
 
         object1 = 'object1'
         client.put_object(self.url, self.token, container, object1, 'test')
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         cpart, cnodes = self.container_ring.get_nodes(self.account, container)
         kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM)
 
         client.delete_object(self.url, self.token, container, object1)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         self.pids[self.port2server[cnodes[1]['port']]] = \
             Popen(['swift-container-server',
@@ -140,9 +140,9 @@ class TestContainerFailures(unittest.TestCase):
                     ((cnodes[1]['port'] - 6001) / 10)]).pid
         sleep(2)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -161,42 +161,42 @@ class TestContainerFailures(unittest.TestCase):
         # account server, this'll pass, otherwise the first account server will
         # serve the listing and not have the container.
         # self.assert_(container in [c['name'] for c in
-        #     client.get_account(self.url, self.token)])
+        #     client.get_account(self.url, self.token)[1]])
 
         object2 = 'object2'
         # This will work because at least one (in this case, just one) account
         # server has to indicate the container exists for the put to continue.
         client.put_object(self.url, self.token, container, object2, 'test')
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         get_to_final_state()
         # Our container delete never "finalized" because we started using it
         # before the delete settled.
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # And, so our object2 should still exist and object1's delete should
         # have finalized.
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
     def test_first_two_nodes_fail(self):
         container = 'container-%s' % uuid4()
         client.put_container(self.url, self.token, container)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
 
         object1 = 'object1'
         client.put_object(self.url, self.token, container, object1, 'test')
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         cpart, cnodes = self.container_ring.get_nodes(self.account, container)
         for x in xrange(2):
@@ -204,9 +204,9 @@ class TestContainerFailures(unittest.TestCase):
 
         client.delete_object(self.url, self.token, container, object1)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         for x in xrange(2):
             self.pids[self.port2server[cnodes[x]['port']]] = \
@@ -215,11 +215,11 @@ class TestContainerFailures(unittest.TestCase):
                         ((cnodes[x]['port'] - 6001) / 10)]).pid
         sleep(2)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # This okay because the first node hasn't got the update that the
         # object was deleted yet.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -238,7 +238,7 @@ class TestContainerFailures(unittest.TestCase):
         # account server, this'll pass, otherwise the first account server will
         # serve the listing and not have the container.
         # self.assert_(container in [c['name'] for c in
-        #              client.get_account(self.url, self.token)])
+        #              client.get_account(self.url, self.token)[1]])
 
         object2 = 'object2'
         # This will work because at least one (in this case, just one) account
@@ -246,35 +246,35 @@ class TestContainerFailures(unittest.TestCase):
         client.put_object(self.url, self.token, container, object2, 'test')
         # First node still doesn't know object1 was deleted yet; this is okay.
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         get_to_final_state()
         # Our container delete never "finalized" because we started using it
         # before the delete settled.
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # And, so our object2 should still exist and object1's delete should
         # have finalized.
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
     def test_last_two_nodes_fail(self):
         container = 'container-%s' % uuid4()
         client.put_container(self.url, self.token, container)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
 
         object1 = 'object1'
         client.put_object(self.url, self.token, container, object1, 'test')
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         cpart, cnodes = self.container_ring.get_nodes(self.account, container)
         for x in (1, 2):
@@ -282,9 +282,9 @@ class TestContainerFailures(unittest.TestCase):
 
         client.delete_object(self.url, self.token, container, object1)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         for x in (1, 2):
             self.pids[self.port2server[cnodes[x]['port']]] = \
@@ -293,9 +293,9 @@ class TestContainerFailures(unittest.TestCase):
                         ((cnodes[x]['port'] - 6001) / 10)]).pid
         sleep(2)
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         # This fails because all three nodes have to indicate deletion before
         # we tell the user it worked. Since the first node 409s (it hasn't got
@@ -314,29 +314,29 @@ class TestContainerFailures(unittest.TestCase):
         # account server, this'll pass, otherwise the first account server will
         # serve the listing and not have the container.
         # self.assert_(container in [c['name'] for c in
-        #     client.get_account(self.url, self.token)])
+        #     client.get_account(self.url, self.token)[1]])
 
         object2 = 'object2'
         # This will work because at least one (in this case, just one) account
         # server has to indicate the container exists for the put to continue.
         client.put_object(self.url, self.token, container, object2, 'test')
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         # And, of course, our new object2 exists.
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
         get_to_final_state()
         # Our container delete never "finalized" because we started using it
         # before the delete settled.
         self.assert_(container in [c['name'] for c in
-                     client.get_account(self.url, self.token)])
+                     client.get_account(self.url, self.token)[1]])
         # And, so our object2 should still exist and object1's delete should
         # have finalized.
         self.assert_(object1 not in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
         self.assert_(object2 in [o['name'] for o in
-                     client.get_container(self.url, self.token, container)])
+                     client.get_container(self.url, self.token, container)[1]])
 
 
 if __name__ == '__main__':

test/probe/test_object_async_update.py

@@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
                     ((cnode['port'] - 6001) / 10)]).pid
         sleep(2)
         self.assert_(not direct_client.direct_get_container(cnode, cpart,
-                            self.account, container))
+                            self.account, container)[1])
         ps = []
         for n in xrange(1, 5):
             ps.append(Popen(['swift-object-updater',
@@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase):
         for p in ps:
             p.wait()
         objs = [o['name'] for o in direct_client.direct_get_container(cnode,
-                                    cpart, self.account, container)]
+                                    cpart, self.account, container)[1]]
         self.assert_(obj in objs)
 
 

test/probe/test_object_handoff.py

@@ -75,13 +75,13 @@ class TestObjectHandoff(unittest.TestCase):
             raise Exception('Direct object GET did not return VERIFY, instead '
                             'it returned: %s' % repr(odata))
         objs = [o['name'] for o in
-                client.get_container(self.url, self.token, container)]
+                client.get_container(self.url, self.token, container)[1]]
         if obj not in objs:
             raise Exception('Container listing did not know about object')
         for cnode in cnodes:
             objs = [o['name'] for o in
                     direct_client.direct_get_container(cnode, cpart,
-                        self.account, container)]
+                        self.account, container)[1]]
             if obj not in objs:
                 raise Exception(
                     'Container server %s:%s did not know about object' %
@@ -126,11 +126,10 @@ class TestObjectHandoff(unittest.TestCase):
 
         kill(self.pids[self.port2server[onode['port']]], SIGTERM)
         client.post_object(self.url, self.token, container, obj,
-                           {'probe': 'value'})
+                           headers={'x-object-meta-probe': 'value'})
-        ometadata = client.head_object(
+        oheaders = client.head_object(self.url, self.token, container, obj)
-            self.url, self.token, container, obj)[-1]
+        if oheaders.get('x-object-meta-probe') != 'value':
-        if ometadata.get('probe') != 'value':
+            raise Exception('Metadata incorrect, was %s' % repr(oheaders))
-            raise Exception('Metadata incorrect, was %s' % repr(ometadata))
         exc = False
         try:
             direct_client.direct_get_object(another_onode, opart, self.account,
@@ -145,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase):
             '/etc/swift/object-server/%d.conf' %
             ((onode['port'] - 6000) / 10)]).pid
         sleep(2)
-        ometadata = direct_client.direct_get_object(onode, opart, self.account,
+        oheaders = direct_client.direct_get_object(onode, opart, self.account,
-                                                    container, obj)[-2]
+                                                    container, obj)[0]
-        if ometadata.get('probe') == 'value':
+        if oheaders.get('x-object-meta-probe') == 'value':
             raise Exception('Previously downed object server had the new '
                             'metadata when it should not have it')
         # Run the extra server last so it'll remove it's extra partition
@@ -161,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase):
         call(['swift-object-replicator',
               '/etc/swift/object-server/%d.conf' %
               ((another_onode['port'] - 6000) / 10), 'once'])
-        ometadata = direct_client.direct_get_object(onode, opart, self.account,
+        oheaders = direct_client.direct_get_object(onode, opart, self.account,
-                                                    container, obj)[-2]
+                                                    container, obj)[0]
-        if ometadata.get('probe') != 'value':
+        if oheaders.get('x-object-meta-probe') != 'value':
             raise Exception(
                 'Previously downed object server did not have the new metadata')
 
@@ -177,13 +176,13 @@ class TestObjectHandoff(unittest.TestCase):
         if not exc:
             raise Exception('Regular object HEAD was still successful')
         objs = [o['name'] for o in
-                client.get_container(self.url, self.token, container)]
+                client.get_container(self.url, self.token, container)[1]]
         if obj in objs:
             raise Exception('Container listing still knew about object')
         for cnode in cnodes:
             objs = [o['name'] for o in
                     direct_client.direct_get_container(
-                        cnode, cpart, self.account, container)]
+                        cnode, cpart, self.account, container)[1]]
             if obj in objs:
                 raise Exception(
                     'Container server %s:%s still knew about object' %

test/probe/test_running_with_each_type_down.py

@@ -56,16 +56,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
             pass
         client.put_object(self.url, self.token, 'container1', 'object1', '1234')
         get_to_final_state()
-        self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
+        headers, containers = client.head_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
-        for container in client.get_account(self.url, self.token):
+        for container in containers:
             if container['name'] == 'container1':
                 found1 = True
                 self.assertEquals(container['count'], 1)
                 self.assertEquals(container['bytes'], 4)
         self.assert_(found1)
         found1 = False
-        for obj in client.get_container(self.url, self.token, 'container1'):
+        for obj in client.get_container(self.url, self.token, 'container1')[1]:
             if obj['name'] == 'object1':
                 found1 = True
                 self.assertEquals(obj['bytes'], 4)
@@ -84,15 +87,18 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
                    '/etc/swift/object-server/%d.conf' %
                     ((onodes[0]['port'] - 6000) / 10)]).pid
         sleep(2)
-        self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
+        headers, containers = client.head_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
-        for container in client.get_account(self.url, self.token):
+        for container in containers:
             if container['name'] == 'container1':
                 found1 = True
         # The account node was previously down.
         self.assert_(not found1)
         found1 = False
-        for obj in client.get_container(self.url, self.token, 'container1'):
+        for obj in client.get_container(self.url, self.token, 'container1')[1]:
             if obj['name'] == 'object1':
                 found1 = True
                 self.assertEquals(obj['bytes'], 4)
@@ -101,16 +107,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase):
         self.assert_(found1)
 
         get_to_final_state()
-        self.assert_(client.head_account(self.url, self.token), (1, 1, 1234))
+        headers, containers = client.head_account(self.url, self.token)
+        self.assertEquals(headers['x-account-container-count'], '1')
+        self.assertEquals(headers['x-account-object-count'], '1')
+        self.assertEquals(headers['x-account-bytes-used'], '4')
         found1 = False
-        for container in client.get_account(self.url, self.token):
+        for container in containers:
             if container['name'] == 'container1':
                 found1 = True
                 self.assertEquals(container['count'], 1)
                 self.assertEquals(container['bytes'], 4)
         self.assert_(found1)
         found1 = False
-        for obj in client.get_container(self.url, self.token, 'container1'):
+        for obj in client.get_container(self.url, self.token, 'container1')[1]:
             if obj['name'] == 'object1':
                 found1 = True
                 self.assertEquals(obj['bytes'], 4)

test/unit/auth/test_server.py

@@ -21,10 +21,11 @@ from StringIO import StringIO
 from uuid import uuid4
 from logging import StreamHandler
 
+import sqlite3
 from webob import Request
 
 from swift.auth import server as auth_server
-from swift.common.db import DatabaseConnectionError
+from swift.common.db import DatabaseConnectionError, get_db_connection
 from swift.common.utils import get_logger
 
 
@@ -106,38 +107,25 @@ class TestAuthServer(unittest.TestCase):
 
     def test_validate_token_non_existant_token(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing',).split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Storage-User': 'tester',
                          'X-Storage-Pass': 'testing'}))
         token = res.headers['x-storage-token']
-        self.assertEquals(self.controller.validate_token(token + 'bad',
+        self.assertEquals(self.controller.validate_token(token + 'bad'), False)
-            cfaccount), False)
-
-    def test_validate_token_non_existant_cfaccount(self):
-        auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
-            'test', 'tester', 'testing').split('/')[-1]
-        res = self.controller.handle_auth(Request.blank('/v1/test/auth',
-                environ={'REQUEST_METHOD': 'GET'},
-                headers={'X-Storage-User': 'tester',
-                         'X-Storage-Pass': 'testing'}))
-        token = res.headers['x-storage-token']
-        self.assertEquals(self.controller.validate_token(token,
-            cfaccount + 'bad'), False)
 
     def test_validate_token_good(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing',).split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Storage-User': 'tester',
                          'X-Storage-Pass': 'testing'}))
         token = res.headers['x-storage-token']
-        ttl = self.controller.validate_token(token, cfaccount)
+        ttl = self.controller.validate_token(token)
         self.assert_(ttl > 0, repr(ttl))
 
     def test_validate_token_expired(self):
@@ -145,40 +133,38 @@ class TestAuthServer(unittest.TestCase):
         try:
             auth_server.time = lambda: 1
             auth_server.http_connect = fake_http_connect(201, 201, 201)
-            cfaccount = self.controller.create_account('test', 'tester',
+            cfaccount = self.controller.create_user('test', 'tester',
                             'testing').split('/')[-1]
             res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                     environ={'REQUEST_METHOD': 'GET'},
                     headers={'X-Storage-User': 'tester',
                              'X-Storage-Pass': 'testing'}))
             token = res.headers['x-storage-token']
-            ttl = self.controller.validate_token(
+            ttl = self.controller.validate_token(token)
-                token, cfaccount)
             self.assert_(ttl > 0, repr(ttl))
             auth_server.time = lambda: 1 + self.controller.token_life
-            self.assertEquals(self.controller.validate_token(
+            self.assertEquals(self.controller.validate_token(token), False)
-                token, cfaccount), False)
         finally:
             auth_server.time = orig_time
 
-    def test_create_account_no_new_account(self):
+    def test_create_user_no_new_account(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        result = self.controller.create_account('', 'tester', 'testing')
+        result = self.controller.create_user('', 'tester', 'testing')
         self.assertFalse(result)
 
-    def test_create_account_no_new_user(self):
+    def test_create_user_no_new_user(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        result = self.controller.create_account('test', '', 'testing')
+        result = self.controller.create_user('test', '', 'testing')
         self.assertFalse(result)
 
-    def test_create_account_no_new_password(self):
+    def test_create_user_no_new_password(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        result = self.controller.create_account('test', 'tester', '')
+        result = self.controller.create_user('test', 'tester', '')
         self.assertFalse(result)
 
-    def test_create_account_good(self):
+    def test_create_user_good(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test', 'tester', 'testing')
+        url = self.controller.create_user('test', 'tester', 'testing')
         self.assert_(url)
         self.assertEquals('/'.join(url.split('/')[:-1]),
             self.controller.default_cluster_url.rstrip('/'), repr(url))
@@ -191,7 +177,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_recreate_accounts_one(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        self.controller.create_account('test', 'tester', 'testing')
+        self.controller.create_user('test', 'tester', 'testing')
         auth_server.http_connect = fake_http_connect(201, 201, 201)
         rv = self.controller.recreate_accounts()
         self.assertEquals(rv.split()[0], '1', repr(rv))
@@ -199,13 +185,13 @@ class TestAuthServer(unittest.TestCase):
 
     def test_recreate_accounts_several(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        self.controller.create_account('test1', 'tester', 'testing')
+        self.controller.create_user('test1', 'tester', 'testing')
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        self.controller.create_account('test2', 'tester', 'testing')
+        self.controller.create_user('test2', 'tester', 'testing')
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        self.controller.create_account('test3', 'tester', 'testing')
+        self.controller.create_user('test3', 'tester', 'testing')
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        self.controller.create_account('test4', 'tester', 'testing')
+        self.controller.create_user('test4', 'tester', 'testing')
         auth_server.http_connect = fake_http_connect(201, 201, 201,
                                                      201, 201, 201,
                                                      201, 201, 201,
@@ -216,7 +202,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_recreate_accounts_one_fail(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test', 'tester', 'testing')
+        url = self.controller.create_user('test', 'tester', 'testing')
         cfaccount = url.split('/')[-1]
         auth_server.http_connect = fake_http_connect(500, 500, 500)
         rv = self.controller.recreate_accounts()
@@ -226,16 +212,16 @@ class TestAuthServer(unittest.TestCase):
 
     def test_recreate_accounts_several_fail(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test1', 'tester', 'testing')
+        url = self.controller.create_user('test1', 'tester', 'testing')
         cfaccounts = [url.split('/')[-1]]
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test2', 'tester', 'testing')
+        url = self.controller.create_user('test2', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test3', 'tester', 'testing')
+        url = self.controller.create_user('test3', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test4', 'tester', 'testing')
+        url = self.controller.create_user('test4', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(500, 500, 500,
                                                      500, 500, 500,
@@ -244,20 +230,20 @@ class TestAuthServer(unittest.TestCase):
         rv = self.controller.recreate_accounts()
         self.assertEquals(rv.split()[0], '4', repr(rv))
         failed = rv.split('[', 1)[-1][:-1].split(', ')
-        self.assertEquals(failed, [repr(a) for a in cfaccounts])
+        self.assertEquals(set(failed), set(repr(a) for a in cfaccounts))
 
     def test_recreate_accounts_several_fail_some(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test1', 'tester', 'testing')
+        url = self.controller.create_user('test1', 'tester', 'testing')
         cfaccounts = [url.split('/')[-1]]
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test2', 'tester', 'testing')
+        url = self.controller.create_user('test2', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test3', 'tester', 'testing')
+        url = self.controller.create_user('test3', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        url = self.controller.create_account('test4', 'tester', 'testing')
+        url = self.controller.create_user('test4', 'tester', 'testing')
         cfaccounts.append(url.split('/')[-1])
         auth_server.http_connect = fake_http_connect(500, 500, 500,
                                                      201, 201, 201,
@@ -266,11 +252,8 @@ class TestAuthServer(unittest.TestCase):
         rv = self.controller.recreate_accounts()
         self.assertEquals(rv.split()[0], '4', repr(rv))
         failed = rv.split('[', 1)[-1][:-1].split(', ')
-        expected = []
+        self.assertEquals(
-        for i, value in enumerate(cfaccounts):
+            len(set(repr(a) for a in cfaccounts) - set(failed)), 2)
-            if not i % 2:
-                expected.append(repr(value))
-        self.assertEquals(failed, expected)
 
     def test_auth_bad_path(self):
         self.assertRaises(ValueError, self.controller.handle_auth,
@@ -281,7 +264,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_SOSO_missing_headers(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -297,7 +280,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_SOSO_bad_account(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/testbad/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -312,7 +295,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_SOSO_bad_user(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -327,7 +310,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_SOSO_bad_password(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -342,31 +325,31 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_SOSO_good(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Storage-User': 'tester',
                          'X-Storage-Pass': 'testing'}))
         token = res.headers['x-storage-token']
-        ttl = self.controller.validate_token(token, cfaccount)
+        ttl = self.controller.validate_token(token)
         self.assert_(ttl > 0, repr(ttl))
 
     def test_auth_SOSO_good_Mosso_headers(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Auth-User': 'test:tester',
                          'X-Auth-Key': 'testing'}))
         token = res.headers['x-storage-token']
-        ttl = self.controller.validate_token(token, cfaccount)
+        ttl = self.controller.validate_token(token)
         self.assert_(ttl > 0, repr(ttl))
 
     def test_auth_SOSO_bad_Mosso_headers(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing',).split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/v1/test/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -386,7 +369,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_missing_headers(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'}))
@@ -402,7 +385,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_bad_header_format(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -417,7 +400,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_bad_account(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -432,7 +415,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_bad_user(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -447,7 +430,7 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_bad_password(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
@@ -462,26 +445,26 @@ class TestAuthServer(unittest.TestCase):
 
     def test_auth_Mosso_good(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Auth-User': 'test:tester',
                          'X-Auth-Key': 'testing'}))
         token = res.headers['x-storage-token']
-        ttl = self.controller.validate_token(token, cfaccount)
+        ttl = self.controller.validate_token(token)
         self.assert_(ttl > 0, repr(ttl))
 
     def test_auth_Mosso_good_SOSO_header_names(self):
         auth_server.http_connect = fake_http_connect(201, 201, 201)
-        cfaccount = self.controller.create_account(
+        cfaccount = self.controller.create_user(
             'test', 'tester', 'testing').split('/')[-1]
         res = self.controller.handle_auth(Request.blank('/auth',
                 environ={'REQUEST_METHOD': 'GET'},
                 headers={'X-Storage-User': 'test:tester',
                          'X-Storage-Pass': 'testing'}))
         token = res.headers['x-storage-token']
-        ttl = self.controller.validate_token(token, cfaccount)
+        ttl = self.controller.validate_token(token)
         self.assert_(ttl > 0, repr(ttl))
 
     def test_basic_logging(self):
@@ -491,10 +474,10 @@ class TestAuthServer(unittest.TestCase):
         logger.logger.addHandler(log_handler)
         try:
             auth_server.http_connect = fake_http_connect(201, 201, 201)
-            url = self.controller.create_account('test', 'tester', 'testing')
+            url = self.controller.create_user('test', 'tester', 'testing')
             self.assertEquals(log.getvalue().rsplit(' ', 1)[0],
-                "auth SUCCESS create_account('test', 'tester', _) = %s" %
+                "auth SUCCESS create_user('test', 'tester', _, False) = %s"
-                repr(url))
+                % repr(url))
             log.truncate(0)
             def start_response(*args):
                 pass
@@ -594,6 +577,58 @@ class TestAuthServer(unittest.TestCase):
             auth_server.Request = orig_Request
             logger.logger.handlers.remove(log_handler)
 
+    def test_upgrading_from_db1(self):
+        swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex
+        os.mkdir(swift_dir)
+        try:
+            # Create db1
+            db_file = os.path.join(swift_dir, 'auth.db')
+            conn = get_db_connection(db_file, okay_to_create=True)
+            conn.execute('''CREATE TABLE IF NOT EXISTS account (
+                            account TEXT, url TEXT, cfaccount TEXT,
+                            user TEXT, password TEXT)''')
+            conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account
+                            ON account (account)''')
+            conn.execute('''CREATE TABLE IF NOT EXISTS token (
+                            cfaccount TEXT, token TEXT, created FLOAT)''')
+            conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount
+                            ON token (cfaccount)''')
+            conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created
+                            ON token (created)''')
+            conn.execute('''INSERT INTO account
+                            (account, url, cfaccount, user, password)
+                            VALUES ('act', 'url', 'cfa', 'usr', 'pas')''')
+            conn.execute('''INSERT INTO token (cfaccount, token, created)
+                            VALUES ('cfa', 'tok', '1')''')
+            conn.commit()
+            conn.close()
+            # Upgrade to current db
+            conf = {'swift_dir': swift_dir}
+            controller = auth_server.AuthController(conf, FakeRing())
+            # Check new items exist and are correct
+            conn = get_db_connection(db_file)
+            row = conn.execute('SELECT admin FROM account').fetchone()
+            self.assertEquals(row[0], 't')
+            row = conn.execute('SELECT user FROM token').fetchone()
+            self.assert_(not row)
+        finally:
+            rmtree(swift_dir)
+
+    def test_create_user_twice(self):
+        auth_server.http_connect = fake_http_connect(201, 201, 201)
+        self.controller.create_user('test', 'tester', 'testing')
+        auth_server.http_connect = fake_http_connect(201, 201, 201)
+        self.assertEquals(
+            self.controller.create_user('test', 'tester', 'testing'),
+            'already exists')
+
+    def test_create_2users_1account(self):
+        auth_server.http_connect = fake_http_connect(201, 201, 201)
+        url = self.controller.create_user('test', 'tester', 'testing')
+        auth_server.http_connect = fake_http_connect(201, 201, 201)
+        url2 = self.controller.create_user('test', 'tester2', 'testing2')
+        self.assertEquals(url, url2)
+
 
 if __name__ == '__main__':
     unittest.main()

test/unit/common/middleware/test_acl.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2010 OpenStack, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from swift.common.middleware import acl
+
+
+class TestACL(unittest.TestCase):
+
+    def test_clean_acl(self):
+        value = acl.clean_acl('header', '.r:*')
+        self.assertEquals(value, '.r:*')
+        value = acl.clean_acl('header', '.r:specific.host')
+        self.assertEquals(value, '.r:specific.host')
+        value = acl.clean_acl('header', '.r:.ending.with')
+        self.assertEquals(value, '.r:.ending.with')
+        value = acl.clean_acl('header', '.r:*.ending.with')
+        self.assertEquals(value, '.r:.ending.with')
+        value = acl.clean_acl('header', '.r:-*.ending.with')
+        self.assertEquals(value, '.r:-.ending.with')
+        value = acl.clean_acl('header', '.r:one,.r:two')
+        self.assertEquals(value, '.r:one,.r:two')
+        value = acl.clean_acl('header', '.r:*,.r:-specific.host')
+        self.assertEquals(value, '.r:*,.r:-specific.host')
+        value = acl.clean_acl('header', '.r:*,.r:-.ending.with')
+        self.assertEquals(value, '.r:*,.r:-.ending.with')
+        value = acl.clean_acl('header', '.r:one,.r:-two')
+        self.assertEquals(value, '.r:one,.r:-two')
+        value = acl.clean_acl('header', '.r:one,.r:-two,account,account:user')
+        self.assertEquals(value, '.r:one,.r:-two,account,account:user')
+        value = acl.clean_acl('header', 'TEST_account')
+        self.assertEquals(value, 'TEST_account')
+        value = acl.clean_acl('header', '.ref:*')
+        self.assertEquals(value, '.r:*')
+        value = acl.clean_acl('header', '.referer:*')
+        self.assertEquals(value, '.r:*')
+        value = acl.clean_acl('header', '.referrer:*')
+        self.assertEquals(value, '.r:*')
+        value = acl.clean_acl('header', 
+                              ' .r : one , ,, .r:two , .r : - three ')
+        self.assertEquals(value, '.r:one,.r:two,.r:-three')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.unknown:test')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:*.')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : * . ')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-*.')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : - * . ')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : ')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : ')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-')
+        self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : - ')
+        self.assertRaises(ValueError, acl.clean_acl, 'header',
+                          'user , .r : - ')
+        self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.r:r')
+
+    def test_parse_acl(self):
+        self.assertEquals(acl.parse_acl(None), ([], []))
+        self.assertEquals(acl.parse_acl(''), ([], []))
+        self.assertEquals(acl.parse_acl('.r:ref1'), (['ref1'], []))
+        self.assertEquals(acl.parse_acl('.r:-ref1'), (['-ref1'], []))
+        self.assertEquals(acl.parse_acl('account:user'),
+                          ([], ['account:user']))
+        self.assertEquals(acl.parse_acl('account'), ([], ['account']))
+        self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.r:ref3,.r:-ref4'),
+                          (['ref3', '-ref4'], ['acc1', 'acc2:usr2']))
+        self.assertEquals(acl.parse_acl(
+            'acc1,acc2:usr2,.r:ref3,acc3,acc4:usr4,.r:ref5,.r:-ref6'),
+            (['ref3', 'ref5', '-ref6'],
+             ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4']))
+
+    def test_referrer_allowed(self):
+        self.assert_(not acl.referrer_allowed('host', None))
+        self.assert_(not acl.referrer_allowed('host', []))
+        self.assert_(acl.referrer_allowed(None, ['*']))
+        self.assert_(acl.referrer_allowed('', ['*']))
+        self.assert_(not acl.referrer_allowed(None, ['specific.host']))
+        self.assert_(not acl.referrer_allowed('', ['specific.host']))
+        self.assert_(acl.referrer_allowed('http://www.example.com/index.html',
+                                          ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://user@www.example.com/index.html', ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://user:pass@www.example.com/index.html', ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://www.example.com:8080/index.html', ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://user@www.example.com:8080/index.html', ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://user:pass@www.example.com:8080/index.html',
+            ['.example.com']))
+        self.assert_(acl.referrer_allowed(
+            'http://user:pass@www.example.com:8080', ['.example.com']))
+        self.assert_(acl.referrer_allowed('http://www.example.com',
+                                          ['.example.com']))
+        self.assert_(not acl.referrer_allowed('http://thief.example.com',
+            ['.example.com', '-thief.example.com']))
+        self.assert_(not acl.referrer_allowed('http://thief.example.com',
+            ['*', '-thief.example.com']))
+        self.assert_(acl.referrer_allowed('http://www.example.com',
+            ['.other.com', 'www.example.com']))
+        self.assert_(acl.referrer_allowed('http://www.example.com',
+            ['-.example.com', 'www.example.com']))
+        # This is considered a relative uri to the request uri, a mode not
+        # currently supported.
+        self.assert_(not acl.referrer_allowed('www.example.com',
+                                              ['.example.com']))
+        self.assert_(not acl.referrer_allowed('../index.html',
+                                              ['.example.com']))
+        self.assert_(acl.referrer_allowed('www.example.com', ['*']))
+
+
+if __name__ == '__main__':
+    unittest.main()

test/unit/common/middleware/test_auth.py

@@ -94,7 +94,7 @@ class Logger(object):
 
 class FakeApp(object):
     def __call__(self, env, start_response):
-        return "OK"
+        return ['204 No Content']
 
 def start_response(*args):
     pass
@@ -102,75 +102,192 @@ def start_response(*args):
 class TestAuth(unittest.TestCase):
 
     def setUp(self):
-        self.test_auth = auth.DevAuthMiddleware(
+        self.test_auth = auth.filter_factory({})(FakeApp())
-            FakeApp(), {}, FakeMemcache(), Logger())
 
     def test_auth_fail(self):
         old_http_connect = auth.http_connect
         try:
             auth.http_connect = mock_http_connect(404)
-            self.assertFalse(self.test_auth.auth('a','t'))
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()},
+                lambda x, y: None))
+            self.assert_(result.startswith('401'), result)
         finally:
             auth.http_connect = old_http_connect
 
     def test_auth_success(self):
         old_http_connect = auth.http_connect
         try:
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+            auth.http_connect = mock_http_connect(204,
-            self.assertTrue(self.test_auth.auth('a','t'))
+               {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()},
+                lambda x, y: None))
+            self.assert_(result.startswith('204'), result)
         finally:
             auth.http_connect = old_http_connect
 
     def test_auth_memcache(self):
         old_http_connect = auth.http_connect
         try:
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+            fake_memcache = FakeMemcache()
-            self.assertTrue(self.test_auth.auth('a','t'))
+            auth.http_connect = mock_http_connect(204,
+               {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
+                lambda x, y: None))
+            self.assert_(result.startswith('204'), result)
             auth.http_connect = mock_http_connect(404)
             # Should still be in memcache
-            self.assertTrue(self.test_auth.auth('a','t'))
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
+                lambda x, y: None))
+            self.assert_(result.startswith('204'), result)
+        finally:
+            auth.http_connect = old_http_connect
+
+    def test_auth_just_expired(self):
+        old_http_connect = auth.http_connect
+        try:
+            fake_memcache = FakeMemcache()
+            auth.http_connect = mock_http_connect(204,
+                {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
+                lambda x, y: None))
+            self.assert_(result.startswith('204'), result)
+            auth.http_connect = mock_http_connect(404)
+            # Should still be in memcache, but expired
+            result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET',
+                'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache},
+                lambda x, y: None))
+            self.assert_(result.startswith('401'), result)
         finally:
             auth.http_connect = old_http_connect
 
     def test_middleware_success(self):
         old_http_connect = auth.http_connect
         try:
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+            auth.http_connect = mock_http_connect(204,
-            req = Request.blank('/v/a/c/o', headers={'x-auth-token':'t'})
+               {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
-            resp = self.test_auth(req.environ, start_response)
+            req = Request.blank('/v/a/c/o', headers={'x-auth-token': 'AUTH_t'})
-            self.assertEquals(resp, 'OK')
+            req.environ['swift.cache'] = FakeMemcache()
+            result = ''.join(self.test_auth(req.environ, start_response))
+            self.assert_(result.startswith('204'), result)
+            self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa')
         finally:
             auth.http_connect = old_http_connect
 
     def test_middleware_no_header(self):
         old_http_connect = auth.http_connect
         try:
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+            auth.http_connect = mock_http_connect(204,
+               {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
             req = Request.blank('/v/a/c/o')
-            resp = self.test_auth(req.environ, start_response)
+            req.environ['swift.cache'] = FakeMemcache()
-            self.assertEquals(resp, ['Missing Auth Token'])
+            result = ''.join(self.test_auth(req.environ, start_response))
+            self.assert_(result.startswith('204'), result)
+            self.assert_(not req.remote_user, req.remote_user)
         finally:
             auth.http_connect = old_http_connect
 
     def test_middleware_storage_token(self):
         old_http_connect = auth.http_connect
         try:
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+            auth.http_connect = mock_http_connect(204,
-            req = Request.blank('/v/a/c/o', headers={'x-storage-token':'t'})
+               {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'})
-            resp = self.test_auth(req.environ, start_response)
+            req = Request.blank('/v/a/c/o',
-            self.assertEquals(resp, 'OK')
+                                headers={'x-storage-token': 'AUTH_t'})
+            req.environ['swift.cache'] = FakeMemcache()
+            result = ''.join(self.test_auth(req.environ, start_response))
+            self.assert_(result.startswith('204'), result)
+            self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa')
         finally:
             auth.http_connect = old_http_connect
 
-    def test_middleware_only_version(self):
+    def test_authorize_bad_path(self):
-        old_http_connect = auth.http_connect
+        req = Request.blank('/badpath')
-        try:
+        resp = str(self.test_auth.authorize(req))
-            auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'})
+        self.assert_(resp.startswith('401'), resp)
-            req = Request.blank('/v', headers={'x-auth-token':'t'})
+        req = Request.blank('/badpath')
-            resp = self.test_auth(req.environ, start_response)
+        req.remote_user = 'act:usr,act,AUTH_cfa'
-            self.assertEquals(resp, ['Bad URL'])
+        resp = str(self.test_auth.authorize(req))
-        finally:
+        self.assert_(resp.startswith('403'), resp)
-            auth.http_connect = old_http_connect
+
+    def test_authorize_account_access(self):
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act,AUTH_cfa'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+
+    def test_authorize_acl_group_access(self):
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = 'act'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = 'act:usr'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = 'act2'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = 'act:usr2'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+
+    def test_deny_cross_reseller(self):
+        # Tests that cross-reseller is denied, even if ACLs/group names match
+        req = Request.blank('/v1/OTHER_cfa')
+        req.remote_user = 'act:usr,act,AUTH_cfa'
+        req.acl = 'act'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+
+    def test_authorize_acl_referrer_access(self):
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = '.r:*'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.acl = '.r:.example.com'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('403'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.remote_user = 'act:usr,act'
+        req.referer = 'http://www.example.com/index.html'
+        req.acl = '.r:.example.com'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('401'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.acl = '.r:*'
+        self.assertEquals(self.test_auth.authorize(req), None)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.acl = '.r:.example.com'
+        resp = str(self.test_auth.authorize(req))
+        self.assert_(resp.startswith('401'), resp)
+        req = Request.blank('/v1/AUTH_cfa')
+        req.referer = 'http://www.example.com/index.html'
+        req.acl = '.r:.example.com'
+        self.assertEquals(self.test_auth.authorize(req), None)
 
 
 if __name__ == '__main__':

test/unit/common/test_client.py

@@ -191,7 +191,7 @@ class TestGetAccount(MockHttpTest):
 
     def test_no_content(self):
         c.http_connection = self.fake_http_connection(204)
-        value = c.get_account('http://www.test.com', 'asdf')
+        value = c.get_account('http://www.test.com', 'asdf')[1]
         self.assertEquals(value, [])
 
 
@@ -200,7 +200,10 @@ class TestHeadAccount(MockHttpTest):
     def test_ok(self):
         c.http_connection = self.fake_http_connection(200)
         value = c.head_account('http://www.tests.com', 'asdf')
-        self.assertEquals(value, (0, 0, 0))
+        # TODO: Hmm. This doesn't really test too much as it uses a fake that
+        # always returns the same dict. I guess it "exercises" the code, so
+        # I'll leave it for now.
+        self.assertEquals(type(value), dict)
 
     def test_server_error(self):
         c.http_connection = self.fake_http_connection(500)
@@ -212,7 +215,7 @@ class TestGetContainer(MockHttpTest):
 
     def test_no_content(self):
         c.http_connection = self.fake_http_connection(204)
-        value = c.get_container('http://www.test.com', 'asdf', 'asdf')
+        value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1]
         self.assertEquals(value, [])
 
 

test/unit/common/test_direct_client.py

@@ -16,7 +16,6 @@
 # TODO: Tests
 
 import unittest
-from swift.common import direct_client
 
 class TestAuditor(unittest.TestCase):
 

test/unit/container/test_server.py

@@ -55,6 +55,49 @@ class TestContainerController(unittest.TestCase):
         """ Tear down for testing swift.object_server.ObjectController """
         rmtree(self.testdir, ignore_errors=1)
 
+    def test_acl_container(self):
+        # Ensure no acl by default
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': '0'})
+        self.controller.PUT(req)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        response = self.controller.HEAD(req)
+        self.assert_(response.status.startswith('204'))
+        self.assert_('x-container-read' not in response.headers)
+        self.assert_('x-container-write' not in response.headers)
+        # Ensure POSTing acls works
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': '1', 'X-Container-Read': '.r:*',
+                     'X-Container-Write': 'account:user'})
+        self.controller.POST(req)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        response = self.controller.HEAD(req)
+        self.assert_(response.status.startswith('204'))
+        self.assertEquals(response.headers.get('x-container-read'), '.r:*')
+        self.assertEquals(response.headers.get('x-container-write'),
+                          'account:user')
+        # Ensure we can clear acls on POST
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'},
+            headers={'X-Timestamp': '3', 'X-Container-Read': '',
+                     'X-Container-Write': ''})
+        self.controller.POST(req)
+        req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'})
+        response = self.controller.HEAD(req)
+        self.assert_(response.status.startswith('204'))
+        self.assert_('x-container-read' not in response.headers)
+        self.assert_('x-container-write' not in response.headers)
+        # Ensure PUTing acls works
+        req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'},
+            headers={'X-Timestamp': '4', 'X-Container-Read': '.r:*',
+                     'X-Container-Write': 'account:user'})
+        self.controller.PUT(req)
+        req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'})
+        response = self.controller.HEAD(req)
+        self.assert_(response.status.startswith('204'))
+        self.assertEquals(response.headers.get('x-container-read'), '.r:*')
+        self.assertEquals(response.headers.get('x-container-write'),
+                          'account:user')
+
     def test_HEAD(self):
         req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
             'HTTP_X_TIMESTAMP': '0'})

test/unit/obj/test_server.py

@@ -19,6 +19,7 @@ import cPickle as pickle
 import os
 import sys
 import unittest
+from nose import SkipTest
 from shutil import rmtree
 from StringIO import StringIO
 from time import gmtime, sleep, strftime, time
@@ -64,7 +65,7 @@ class TestObjectController(unittest.TestCase):
     def test_POST_update_meta(self):
         """ Test swift.object_server.ObjectController.POST """
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'X-Timestamp': timestamp,
@@ -92,7 +93,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_POST_not_exist(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'},
                             headers={'X-Timestamp': timestamp,
@@ -114,7 +115,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_POST_container_connection(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         def mock_http_connect(response, with_exc=False):
             class FakeConn(object):
                 def __init__(self, status, with_exc):
@@ -210,7 +211,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_PUT_common(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': timestamp,
@@ -234,7 +235,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_PUT_overwrite(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': normalize_timestamp(time()),
                          'Content-Length': '6',
@@ -267,7 +268,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_PUT_no_etag(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={'X-Timestamp': normalize_timestamp(time()),
                                      'Content-Type': 'text/plain'})
@@ -286,7 +287,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_PUT_user_metadata(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                 headers={'X-Timestamp': timestamp,
@@ -314,7 +315,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_PUT_container_connection(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         def mock_http_connect(response, with_exc=False):
             class FakeConn(object):
                 def __init__(self, status, with_exc):
@@ -376,7 +377,7 @@ class TestObjectController(unittest.TestCase):
     def test_HEAD(self):
         """ Test swift.object_server.ObjectController.HEAD """
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c')
         resp = self.object_controller.HEAD(req)
         self.assertEquals(resp.status_int, 400)
@@ -443,7 +444,7 @@ class TestObjectController(unittest.TestCase):
     def test_GET(self):
         """ Test swift.object_server.ObjectController.GET """
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c')
         resp = self.object_controller.GET(req)
         self.assertEquals(resp.status_int, 400)
@@ -532,7 +533,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_GET_if_match(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
                                 'X-Timestamp': normalize_timestamp(time()),
@@ -586,7 +587,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_GET_if_none_match(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
                                 'X-Timestamp': normalize_timestamp(time()),
@@ -637,7 +638,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_GET_if_modified_since(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
@@ -674,7 +675,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_GET_if_unmodified_since(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                             headers={
@@ -713,7 +714,7 @@ class TestObjectController(unittest.TestCase):
     def test_DELETE(self):
         """ Test swift.object_server.ObjectController.DELETE """
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'})
         resp = self.object_controller.DELETE(req)
         self.assertEquals(resp.status_int, 400)
@@ -840,7 +841,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_chunked_put(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         listener = listen(('localhost', 0))
         port = listener.getsockname()[1]
         killer = spawn(wsgi.server, listener, self.object_controller,
@@ -866,7 +867,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_max_object_name_length(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         timestamp = normalize_timestamp(time())
         req = Request.blank('/sda1/p/a/c/' + ('1' * 1024),
                 environ={'REQUEST_METHOD': 'PUT'},
@@ -887,7 +888,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_disk_file_app_iter_corners(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o')
         mkdirs(df.datadir)
         f = open(os.path.join(df.datadir,
@@ -920,7 +921,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_max_upload_time(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         class SlowBody():
             def __init__(self):
                 self.sent = 0
@@ -962,7 +963,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_bad_sinces(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
             headers={'X-Timestamp': normalize_timestamp(time()),
                      'Content-Length': '4', 'Content-Type': 'text/plain'},
@@ -988,7 +989,7 @@ class TestObjectController(unittest.TestCase):
 
     def test_content_encoding(self):
         if not self.path_to_test_xfs:
-            return
+            raise SkipTest
         req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
             headers={'X-Timestamp': normalize_timestamp(time()),
                      'Content-Length': '4', 'Content-Type': 'text/plain',

test/unit/proxy/test_server.py

@@ -19,6 +19,7 @@ import logging
 import os
 import sys
 import unittest
+from nose import SkipTest
 from ConfigParser import ConfigParser
 from contextlib import contextmanager
 from cStringIO import StringIO
@@ -33,6 +34,7 @@ from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen
 from eventlet.timeout import Timeout
 import simplejson
 from webob import Request
+from webob.exc import HTTPUnauthorized
 
 from test.unit import connect_tcp, readuntil2crlfs
 from swift.proxy import server as proxy_server
@@ -81,7 +83,7 @@ def fake_http_connect(*code_iter, **kwargs):
                 pass
             if 'slow' in kwargs:
                 headers['content-length'] = '4'
-            return headers
+            return headers.items()
         def read(self, amt=None):
             if 'slow' in kwargs:
                 if self.sent < 4:
@@ -97,7 +99,7 @@ def fake_http_connect(*code_iter, **kwargs):
                     self.received += 1
                     sleep(0.1)
         def getheader(self, name, default=None):
-            return self.getheaders().get(name.lower(), default)
+            return dict(self.getheaders()).get(name.lower(), default)
     etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter))
     x = kwargs.get('missing_container', [False] * len(code_iter))
     if not isinstance(x, (tuple, list)):
@@ -203,10 +205,39 @@ class TestProxyServer(unittest.TestCase):
         app = MyApp(None, FakeMemcache(), account_ring=FakeRing(),
                 container_ring=FakeRing(), object_ring=FakeRing())
         req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
-        req.account = 'account'
+        app.update_request(req)
         resp = app.handle_request(req)
         self.assertEquals(resp.status_int, 500)
 
+    def test_calls_authorize_allow(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+        with save_globals():
+            proxy_server.http_connect = fake_http_connect(200)
+            app = proxy_server.Application(None, FakeMemcache(),
+                account_ring=FakeRing(), container_ring=FakeRing(),
+                object_ring=FakeRing())
+            req = Request.blank('/v1/a')
+            req.environ['swift.authorize'] = authorize
+            app.update_request(req)
+            resp = app.handle_request(req)
+        self.assert_(called[0])
+
+    def test_calls_authorize_deny(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        app = proxy_server.Application(None, FakeMemcache(),
+            account_ring=FakeRing(), container_ring=FakeRing(),
+            object_ring=FakeRing())
+        req = Request.blank('/v1/a')
+        req.environ['swift.authorize'] = authorize
+        app.update_request(req)
+        resp = app.handle_request(req)
+        self.assert_(called[0])
+
 
 class TestObjectController(unittest.TestCase):
 
@@ -224,14 +255,14 @@ class TestObjectController(unittest.TestCase):
             self.app.memcache.store = {}
             req = Request.blank('/a/c/o', headers={'Content-Length': '0',
                     'Content-Type': 'text/plain'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
             proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
             self.app.memcache.store = {}
             req = Request.blank('/a/c/o', headers={'Content-Length': '0',
                     'Content-Type': 'text/plain'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
 
@@ -244,7 +275,7 @@ class TestObjectController(unittest.TestCase):
                     give_content_type=lambda content_type:
                         self.assertEquals(content_type, expected.next()))
                 req = Request.blank('/a/c/%s' % filename, {})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.PUT(req)
             test_content_type('test.jpg',
                               iter(['', '', '', 'image/jpeg', 'image/jpeg', 'image/jpeg']))
@@ -261,7 +292,7 @@ class TestObjectController(unittest.TestCase):
                 proxy_server.http_connect = fake_http_connect(*statuses)
                 req = Request.blank('/a/c/o.jpg', {})
                 req.content_length = 0
-                req.account = 'a'
+                self.app.update_request(req)
                 self.app.memcache.store = {}
                 res = controller.PUT(req)
                 expected = str(expected)
@@ -296,7 +327,7 @@ class TestObjectController(unittest.TestCase):
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c/o.jpg', {})
                 req.content_length = 0
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.PUT(req)
                 expected = str(expected)
                 self.assertEquals(res.status[:len(expected)], expected)
@@ -330,7 +361,7 @@ class TestObjectController(unittest.TestCase):
                 self.app.memcache.store = {}
                 proxy_server.http_connect = mock_http_connect(*statuses)
                 req = Request.blank('/a/c/o.jpg', {})
-                req.account = 'a'
+                self.app.update_request(req)
                 req.body_file = StringIO('some data')
                 res = controller.PUT(req)
                 expected = str(expected)
@@ -347,7 +378,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', {}, headers={
                 'Content-Length': str(MAX_FILE_SIZE + 1),
                 'Content-Type': 'foo/bar'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.PUT(req)
             self.assertEquals(res.status_int, 413)
 
@@ -379,7 +410,7 @@ class TestObjectController(unittest.TestCase):
                 proxy_server.http_connect = mock_http_connect(*statuses)
                 req = Request.blank('/a/c/o.jpg', {})
                 req.content_length = 0
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.PUT(req)
                 expected = str(expected)
                 self.assertEquals(res.status[:len(str(expected))],
@@ -397,7 +428,7 @@ class TestObjectController(unittest.TestCase):
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c/o', {}, headers={
                                                 'Content-Type': 'foo/bar'})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.POST(req)
                 expected = str(expected)
                 self.assertEquals(res.status[:len(expected)], expected)
@@ -417,7 +448,7 @@ class TestObjectController(unittest.TestCase):
                 proxy_server.http_connect = fake_http_connect(*statuses)
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c/o', {})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.DELETE(req)
                 self.assertEquals(res.status[:len(str(expected))],
                                   str(expected))
@@ -436,7 +467,7 @@ class TestObjectController(unittest.TestCase):
                 proxy_server.http_connect = fake_http_connect(*statuses)
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c/o', {})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.HEAD(req)
                 self.assertEquals(res.status[:len(str(expected))],
                                   str(expected))
@@ -460,14 +491,14 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', {}, headers={
                                             'Content-Type': 'foo/bar',
                                             'X-Object-Meta-Foo': 'x'*256})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 202)
             proxy_server.http_connect = fake_http_connect(202, 202, 202)
             req = Request.blank('/a/c/o', {}, headers={
                                             'Content-Type': 'foo/bar',
                                             'X-Object-Meta-Foo': 'x'*257})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
@@ -481,14 +512,14 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', {}, headers={
                                             'Content-Type': 'foo/bar',
                                             ('X-Object-Meta-'+'x'*128): 'x'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 202)
             proxy_server.http_connect = fake_http_connect(202, 202, 202)
             req = Request.blank('/a/c/o', {}, headers={
                                             'Content-Type': 'foo/bar',
                                             ('X-Object-Meta-'+'x'*129): 'x'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
@@ -500,7 +531,7 @@ class TestObjectController(unittest.TestCase):
             headers.update({'Content-Type': 'foo/bar'})
             proxy_server.http_connect = fake_http_connect(202, 202, 202)
             req = Request.blank('/a/c/o', {}, headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
@@ -512,7 +543,7 @@ class TestObjectController(unittest.TestCase):
             headers.update({'Content-Type': 'foo/bar'})
             proxy_server.http_connect = fake_http_connect(202, 202, 202)
             req = Request.blank('/a/c/o', {}, headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.POST(req)
             self.assertEquals(res.status_int, 400)
 
@@ -542,7 +573,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o',
                 environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
                 headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = \
@@ -554,7 +585,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o',
                 environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
                 headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
-            req.account = 'account'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(201, 201, 201)
                 #                 obj  obj  obj
@@ -583,7 +614,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o',
                 environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()},
                 headers={'Content-Length': '4', 'Content-Type': 'text/plain'})
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = \
@@ -607,7 +638,7 @@ class TestObjectController(unittest.TestCase):
                 dev['ip'] = '127.0.0.1'
                 dev['port'] = 1
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = \
@@ -649,7 +680,7 @@ class TestObjectController(unittest.TestCase):
                 environ={'REQUEST_METHOD': 'PUT'},
                 headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
                 body='    ')
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = \
@@ -663,7 +694,7 @@ class TestObjectController(unittest.TestCase):
                 environ={'REQUEST_METHOD': 'PUT'},
                 headers={'Content-Length': '4', 'Content-Type': 'text/plain'},
                 body='    ')
-            req.account = 'account'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 503)
 
@@ -708,7 +739,7 @@ class TestObjectController(unittest.TestCase):
     def test_proxy_passes_content_type(self):
         with save_globals():
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'})
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = fake_http_connect(200, 200, 200)
@@ -728,7 +759,7 @@ class TestObjectController(unittest.TestCase):
     def test_proxy_passes_content_length_on_head(self):
         with save_globals():
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'})
-            req.account = 'account'
+            self.app.update_request(req)
             controller = proxy_server.ObjectController(self.app, 'account',
                                                        'container', 'object')
             proxy_server.http_connect = fake_http_connect(200, 200, 200)
@@ -777,7 +808,7 @@ class TestObjectController(unittest.TestCase):
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 200, 200, 200, 200)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = getattr(controller, 'DELETE')(req)
             self.assertEquals(resp.status_int, 200)
 
@@ -853,7 +884,7 @@ class TestObjectController(unittest.TestCase):
             proxy_server.http_connect = \
                 fake_http_connect(404, 404, 404, 200, 200, 200)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 404)
 
@@ -861,7 +892,7 @@ class TestObjectController(unittest.TestCase):
                 fake_http_connect(404, 404, 404, 200, 200, 200)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
                                 headers={'Content-Type': 'text/plain'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 404)
 
@@ -874,7 +905,7 @@ class TestObjectController(unittest.TestCase):
                 #                 acct cont obj  obj  obj
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
 
@@ -883,7 +914,7 @@ class TestObjectController(unittest.TestCase):
                 headers={'Content-Length': '0',
                          'X-Object-Meta-' + ('a' *
                             MAX_META_NAME_LENGTH) : 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -891,7 +922,7 @@ class TestObjectController(unittest.TestCase):
                 headers={'Content-Length': '0',
                          'X-Object-Meta-' + ('a' *
                             (MAX_META_NAME_LENGTH + 1)) : 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -900,7 +931,7 @@ class TestObjectController(unittest.TestCase):
                 headers={'Content-Length': '0',
                          'X-Object-Meta-Too-Long': 'a' *
                             MAX_META_VALUE_LENGTH})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -908,7 +939,7 @@ class TestObjectController(unittest.TestCase):
                 headers={'Content-Length': '0',
                          'X-Object-Meta-Too-Long': 'a' *
                             (MAX_META_VALUE_LENGTH + 1)})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -918,7 +949,7 @@ class TestObjectController(unittest.TestCase):
                 headers['X-Object-Meta-%d' % x] = 'v'
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -927,7 +958,7 @@ class TestObjectController(unittest.TestCase):
                 headers['X-Object-Meta-%d' % x] = 'v'
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -946,7 +977,7 @@ class TestObjectController(unittest.TestCase):
                     'a' * (MAX_META_OVERALL_SIZE - size - 1)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -954,7 +985,7 @@ class TestObjectController(unittest.TestCase):
                 'a' * (MAX_META_OVERALL_SIZE - size)
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.PUT(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -964,7 +995,7 @@ class TestObjectController(unittest.TestCase):
                                                        'container', 'object')
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 201, 201, 201)
                 #                 acct cont obj  obj  obj
@@ -974,7 +1005,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': 'c/o'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
                 #                 acct cont acct cont objc obj  obj  obj
@@ -986,7 +1017,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': '/c/o'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201)
                 #                 acct cont acct cont objc obj  obj  obj
@@ -998,7 +1029,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': '/c/o'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 503, 503, 503)
                 #                 acct cont objc objc objc
@@ -1009,7 +1040,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': '/c/o'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 404, 404, 404)
                 #                 acct cont objc objc objc
@@ -1020,7 +1051,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': '/c/o'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201)
                 #                 acct cont objc objc objc obj  obj  obj
@@ -1032,7 +1063,7 @@ class TestObjectController(unittest.TestCase):
                                 headers={'Content-Length': '0',
                                           'X-Copy-From': '/c/o',
                                           'X-Object-Meta-Ours': 'okay'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = \
                 fake_http_connect(200, 200, 200, 201, 201, 201)
                 #                 acct cont objc obj  obj  obj
@@ -1052,7 +1083,7 @@ class TestObjectController(unittest.TestCase):
                 'pointing to a valid directory.\n' \
                 'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \
                 'system for testing.'
-            return
+            raise SkipTest
         testdir = \
             os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked')
         mkdirs(testdir)
@@ -1434,7 +1465,7 @@ class TestObjectController(unittest.TestCase):
                                                        'container', 'object')
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '0'})
-            req.account = 'a'
+            self.app.update_request(req)
             proxy_server.http_connect = fake_http_connect(200, 201, 201, 201,
                 etags=[None,
                        '68b329da9893e34099c7d8ad5cb9c940',
@@ -1452,7 +1483,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '10'},
                                 body='1234567890')
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.PUT(req)
             self.assert_(hasattr(req, 'bytes_transferred'))
             self.assertEquals(req.bytes_transferred, 10)
@@ -1464,7 +1495,7 @@ class TestObjectController(unittest.TestCase):
             controller = proxy_server.ObjectController(self.app, 'account',
                             'container', 'object')
             req = Request.blank('/a/c/o')
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.GET(req)
             res.body
             self.assert_(hasattr(res, 'bytes_transferred'))
@@ -1479,7 +1510,7 @@ class TestObjectController(unittest.TestCase):
             req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
                                 headers={'Content-Length': '10'},
                                 body='12345')
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.PUT(req)
             self.assertEquals(req.bytes_transferred, 5)
             self.assert_(hasattr(req, 'client_disconnect'))
@@ -1492,7 +1523,7 @@ class TestObjectController(unittest.TestCase):
             controller = proxy_server.ObjectController(self.app, 'account',
                             'container', 'object')
             req = Request.blank('/a/c/o')
-            req.account = 'a'
+            self.app.update_request(req)
             orig_object_chunk_size = self.app.object_chunk_size
             try:
                 self.app.object_chunk_size = 5
@@ -1509,6 +1540,73 @@ class TestObjectController(unittest.TestCase):
             finally: 
                 self.app.object_chunk_size = orig_object_chunk_size
 
+    def test_GET_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 201, 201, 201)
+            controller = proxy_server.ObjectController(self.app, 'account',
+                            'container', 'object')
+            req = Request.blank('/a/c/o')
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.GET(req)
+        self.assert_(called[0])
+
+    def test_HEAD_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 201, 201, 201)
+            controller = proxy_server.ObjectController(self.app, 'account',
+                            'container', 'object')
+            req = Request.blank('/a/c/o', {'REQUEST_METHOD': 'HEAD'})
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.HEAD(req)
+        self.assert_(called[0])
+
+    def test_POST_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 201, 201, 201)
+            controller = proxy_server.ObjectController(self.app, 'account',
+                            'container', 'object')
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'},
+                                headers={'Content-Length': '5'}, body='12345')
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.POST(req)
+        self.assert_(called[0])
+
+    def test_PUT_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 200, 201, 201, 201)
+            controller = proxy_server.ObjectController(self.app, 'account',
+                            'container', 'object')
+            req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+                                headers={'Content-Length': '5'}, body='12345')
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.PUT(req)
+        self.assert_(called[0])
+
+
 
 class TestContainerController(unittest.TestCase):
     "Test swift.proxy_server.ContainerController"
@@ -1528,14 +1626,14 @@ class TestContainerController(unittest.TestCase):
             self.app.memcache.store = {}
             req = Request.blank('/a/c', headers={'Content-Length': '0',
                     'Content-Type': 'text/plain'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
             proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
             self.app.memcache.store = {}
             req = Request.blank('/a/c/', headers={'Content-Length': '0',
                     'Content-Type': 'text/plain'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
 
@@ -1547,7 +1645,7 @@ class TestContainerController(unittest.TestCase):
                 proxy_server.http_connect = fake_http_connect(*statuses, **kwargs)
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c', {})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.HEAD(req)
                 self.assertEquals(res.status[:len(str(expected))],
                                   str(expected))
@@ -1570,7 +1668,7 @@ class TestContainerController(unittest.TestCase):
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c', {})
                 req.content_length = 0
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.PUT(req)
                 expected = str(expected)
                 self.assertEquals(res.status[:len(expected)], expected)
@@ -1613,7 +1711,7 @@ class TestContainerController(unittest.TestCase):
                         fake_http_connect(200, 200, 200, 200)
                 self.app.memcache.store = {}
                 req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth})
-                req.account = 'a'
+                self.app.update_request(req)
                 resp = getattr(controller, meth)(req)
                 self.assertEquals(resp.status_int, 200)
 
@@ -1657,7 +1755,7 @@ class TestContainerController(unittest.TestCase):
             self.app.memcache = MockMemcache(allow_lock=True)
             proxy_server.http_connect = fake_http_connect(200, 200, 200, 201, 201, 201, missing_container=True)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'})
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.PUT(req)
             self.assertEquals(res.status_int, 201)
 
@@ -1703,7 +1801,7 @@ class TestContainerController(unittest.TestCase):
             controller = proxy_server.ContainerController(self.app, 'account',
                                                           'container')
             req = Request.blank('/a/c?format=json')
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.GET(req)
             res.body
             self.assert_(hasattr(res, 'bytes_transferred'))
@@ -1715,7 +1813,7 @@ class TestContainerController(unittest.TestCase):
             controller = proxy_server.ContainerController(self.app, 'account',
                                                           'container')
             req = Request.blank('/a/c?format=json')
-            req.account = 'a'
+            self.app.update_request(req)
             orig_object_chunk_size = self.app.object_chunk_size
             try:
                 self.app.object_chunk_size = 1
@@ -1760,8 +1858,7 @@ class TestContainerController(unittest.TestCase):
                     201, give_connect=test_connect)
                 req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                         headers={test_header: test_value})
-                req.account = 'a'
+                self.app.update_request(req)
-                req.container = 'c'
                 res = getattr(controller, method)(req)
                 self.assertEquals(test_errors, [])
 
@@ -1776,7 +1873,7 @@ class TestContainerController(unittest.TestCase):
             controller = proxy_server.ContainerController(self.app, 'a', 'c')
             proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 201)
 
@@ -1784,16 +1881,14 @@ class TestContainerController(unittest.TestCase):
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                     headers={'X-Container-Meta-' +
                                 ('a' * MAX_META_NAME_LENGTH): 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                     headers={'X-Container-Meta-' +
                                 ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -1801,16 +1896,14 @@ class TestContainerController(unittest.TestCase):
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                     headers={'X-Container-Meta-Too-Long':
                                 'a' * MAX_META_VALUE_LENGTH})
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                     headers={'X-Container-Meta-Too-Long':
                                 'a' * (MAX_META_VALUE_LENGTH + 1)})
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -1820,8 +1913,7 @@ class TestContainerController(unittest.TestCase):
                 headers['X-Container-Meta-%d' % x] = 'v'
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -1830,8 +1922,7 @@ class TestContainerController(unittest.TestCase):
                 headers['X-Container-Meta-%d' % x] = 'v'
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -1849,8 +1940,7 @@ class TestContainerController(unittest.TestCase):
                     'a' * (MAX_META_OVERALL_SIZE - size - 1)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 201)
             proxy_server.http_connect = fake_http_connect(201, 201, 201)
@@ -1858,11 +1948,96 @@ class TestContainerController(unittest.TestCase):
                 'a' * (MAX_META_OVERALL_SIZE - size)
             req = Request.blank('/a/c', environ={'REQUEST_METHOD': method},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
-            req.container = 'c'
             resp = getattr(controller, method)(req)
             self.assertEquals(resp.status_int, 400)
 
+    def test_POST_calls_clean_acl(self):
+        called = [False]
+        def clean_acl(header, value):
+            called[0] = True
+            raise ValueError('fake error')
+        with save_globals():
+            proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
+                                headers={'X-Container-Read': '.r:*'})
+            req.environ['swift.clean_acl'] = clean_acl
+            self.app.update_request(req)
+            res = controller.POST(req)
+        self.assert_(called[0])
+        called[0] = False
+        with save_globals():
+            proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'},
+                                headers={'X-Container-Write': '.r:*'})
+            req.environ['swift.clean_acl'] = clean_acl
+            self.app.update_request(req)
+            res = controller.POST(req)
+        self.assert_(called[0])
+
+    def test_PUT_calls_clean_acl(self):
+        called = [False]
+        def clean_acl(header, value):
+            called[0] = True
+            raise ValueError('fake error')
+        with save_globals():
+            proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                                headers={'X-Container-Read': '.r:*'})
+            req.environ['swift.clean_acl'] = clean_acl
+            self.app.update_request(req)
+            res = controller.PUT(req)
+        self.assert_(called[0])
+        called[0] = False
+        with save_globals():
+            proxy_server.http_connect = fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'},
+                                headers={'X-Container-Write': '.r:*'})
+            req.environ['swift.clean_acl'] = clean_acl
+            self.app.update_request(req)
+            res = controller.PUT(req)
+        self.assert_(called[0])
+
+    def test_GET_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c')
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.GET(req)
+        self.assert_(called[0])
+
+    def test_HEAD_calls_authorize(self):
+        called = [False]
+        def authorize(req):
+            called[0] = True
+            return HTTPUnauthorized(request=req)
+        with save_globals():
+            proxy_server.http_connect = \
+                fake_http_connect(200, 201, 201, 201)
+            controller = proxy_server.ContainerController(self.app, 'account',
+                                                          'container')
+            req = Request.blank('/a/c', {'REQUEST_METHOD': 'HEAD'})
+            req.environ['swift.authorize'] = authorize
+            self.app.update_request(req)
+            res = controller.HEAD(req)
+        self.assert_(called[0])
+
 
 class TestAccountController(unittest.TestCase):
 
@@ -1875,12 +2050,12 @@ class TestAccountController(unittest.TestCase):
         with save_globals():
             proxy_server.http_connect = fake_http_connect(*statuses)
             req = Request.blank('/a', {})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
             proxy_server.http_connect = fake_http_connect(*statuses)
             req = Request.blank('/a/', {})
-            req.account = 'a'
+            self.app.update_request(req)
             res = method(req)
             self.assertEquals(res.status_int, expected)
 
@@ -1930,7 +2105,7 @@ class TestAccountController(unittest.TestCase):
             dev['port'] = 1 ## can't connect on this port
         controller = proxy_server.AccountController(self.app, 'account')
         req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
-        req.account = 'account'
+        self.app.update_request(req)
         resp = controller.HEAD(req)
         self.assertEquals(resp.status_int, 503)
 
@@ -1941,7 +2116,7 @@ class TestAccountController(unittest.TestCase):
             dev['port'] = -1 ## invalid port number
         controller = proxy_server.AccountController(self.app, 'account')
         req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'})
-        req.account = 'account'
+        self.app.update_request(req)
         resp = controller.HEAD(req)
         self.assertEquals(resp.status_int, 503)
 
@@ -1950,7 +2125,7 @@ class TestAccountController(unittest.TestCase):
             proxy_server.http_connect = fake_http_connect(200, 200, body='{}')
             controller = proxy_server.AccountController(self.app, 'account')
             req = Request.blank('/a?format=json')
-            req.account = 'a'
+            self.app.update_request(req)
             res = controller.GET(req)
             res.body
             self.assert_(hasattr(res, 'bytes_transferred'))
@@ -1961,7 +2136,7 @@ class TestAccountController(unittest.TestCase):
             proxy_server.http_connect = fake_http_connect(200, 200, body='{}')
             controller = proxy_server.AccountController(self.app, 'account')
             req = Request.blank('/a?format=json')
-            req.account = 'a'
+            self.app.update_request(req)
             orig_object_chunk_size = self.app.object_chunk_size
             try:
                 self.app.object_chunk_size = 1
@@ -1999,7 +2174,7 @@ class TestAccountController(unittest.TestCase):
                     give_connect=test_connect)
                 req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                         headers={test_header: test_value})
-                req.account = 'a'
+                self.app.update_request(req)
                 res = controller.POST(req)
                 self.assertEquals(test_errors, [])
 
@@ -2008,7 +2183,7 @@ class TestAccountController(unittest.TestCase):
             controller = proxy_server.AccountController(self.app, 'a')
             proxy_server.http_connect = fake_http_connect(204, 204, 204)
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 204)
 
@@ -2016,14 +2191,14 @@ class TestAccountController(unittest.TestCase):
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                     headers={'X-Account-Meta-' +
                                 ('a' * MAX_META_NAME_LENGTH): 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 204)
             proxy_server.http_connect = fake_http_connect(204, 204, 204)
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                     headers={'X-Account-Meta-' +
                                 ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -2031,14 +2206,14 @@ class TestAccountController(unittest.TestCase):
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                     headers={'X-Account-Meta-Too-Long':
                                 'a' * MAX_META_VALUE_LENGTH})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 204)
             proxy_server.http_connect = fake_http_connect(204, 204, 204)
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                     headers={'X-Account-Meta-Too-Long':
                                 'a' * (MAX_META_VALUE_LENGTH + 1)})
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -2048,7 +2223,7 @@ class TestAccountController(unittest.TestCase):
                 headers['X-Account-Meta-%d' % x] = 'v'
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 204)
             proxy_server.http_connect = fake_http_connect(204, 204, 204)
@@ -2057,7 +2232,7 @@ class TestAccountController(unittest.TestCase):
                 headers['X-Account-Meta-%d' % x] = 'v'
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 400)
 
@@ -2075,7 +2250,7 @@ class TestAccountController(unittest.TestCase):
                     'a' * (MAX_META_OVERALL_SIZE - size - 1)
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 204)
             proxy_server.http_connect = fake_http_connect(204, 204, 204)
@@ -2083,7 +2258,7 @@ class TestAccountController(unittest.TestCase):
                 'a' * (MAX_META_OVERALL_SIZE - size)
             req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'},
                                 headers=headers)
-            req.account = 'a'
+            self.app.update_request(req)
             resp = controller.POST(req)
             self.assertEquals(resp.status_int, 400)