From cfcfdd3de1108db25e765c4e725a1a048494e8c7 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 2 Sep 2010 21:50:16 -0700 Subject: [PATCH 01/16] Refactored auth and adding ACLs using repoze.what --- bin/swift-auth-create-account | 33 ++- doc/source/development_saio.rst | 12 + swift/auth/server.py | 144 ++++++---- swift/common/middleware/auth.py | 319 ++++++++++++++++----- swift/container/server.py | 9 +- swift/proxy/server.py | 157 ++++++++-- test/functional/tests.py | 16 +- test/functionalnosetests/swift_testing.py | 67 +++-- test/functionalnosetests/test_container.py | 224 ++++++++++++++- test/functionalnosetests/test_object.py | 90 ++++++ test/unit/auth/test_server.py | 44 +-- test/unit/common/middleware/test_auth.py | 119 ++++---- test/unit/container/test_server.py | 45 +++ test/unit/proxy/test_server.py | 177 ++++++------ 14 files changed, 1087 insertions(+), 369 deletions(-) create mode 100644 test/functionalnosetests/test_object.py diff --git a/bin/swift-auth-create-account b/bin/swift-auth-create-account index bab2d3daea..a96e175529 100755 --- a/bin/swift-auth-create-account +++ b/bin/swift-auth-create-account @@ -15,6 +15,7 @@ # limitations under the License. from ConfigParser import ConfigParser +from os.path import basename from sys import argv, exit from swift.common.bufferedhttp import http_connect_raw as http_connect @@ -22,11 +23,27 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect if __name__ == '__main__': f = '/etc/swift/auth-server.conf' - if len(argv) == 5: - f = argv[4] - elif len(argv) != 4: - exit('Syntax: %s [conf_file]' % - argv[0]) + good = False + noaccess = False + if len(argv) == 6 and argv[4] == 'noaccess': + good = True + noaccess = True + f = argv[5] + elif len(argv) == 5: + good = True + if argv[4] == 'noaccess': + noaccess = True + else: + f = argv[4] + elif len(argv) == 4: + good = True + if not good: + exit(''' +Syntax: %s [noaccess] [conf_file] +The noaccess keyword will create a user with no access to the account; another +user for the account will have to add the user to the ACLs for a container to +grant some access. + '''.strip() % basename(argv[0])) new_account = argv[1] new_user = argv[2] new_password = argv[3] @@ -38,8 +55,10 @@ if __name__ == '__main__': port = int(conf.get('bind_port', 11000)) ssl = conf.get('cert_file') is not None path = '/account/%s/%s' % (new_account, new_user) - conn = http_connect(host, port, 'PUT', path, {'x-auth-key':new_password}, - ssl=ssl) + headers = {'X-Auth-Key': new_password} + if noaccess: + headers['X-User-No-Access'] = 'true' + conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl) resp = conn.getresponse() if resp.status == 204: print resp.getheader('x-storage-url') diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 1eaaec5bbe..30e8d8cfc5 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -530,16 +530,28 @@ good idea what to do on other environments. #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` + #. `swift-auth-create-account test2 tester2 testing2` + #. `swift-auth-create-account test tester3 testing3 noaccess` #. Create `/etc/swift/func_test.conf`:: auth_host = 127.0.0.1 auth_port = 11000 auth_ssl = no + # Primary functional test account account = test username = tester password = testing + # User on a second account + account2 = test2 + username2 = tester2 + password2 = testing2 + + # User on same account as first, but with noaccess + username3 = tester3 + password3 = testing3 + collate = C #. `cd ~/swift/trunk; ./.functests` diff --git a/swift/auth/server.py b/swift/auth/server.py index 5658790c88..db290a72c4 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -22,6 +22,7 @@ from time import gmtime, strftime, time from urllib import unquote, quote from uuid import uuid4 +import sqlite3 from webob import Request, Response from webob.exc import HTTPBadRequest, HTTPNoContent, HTTPUnauthorized, \ HTTPServiceUnavailable, HTTPNotFound @@ -58,10 +59,10 @@ class AuthController(object): * The user makes a ReST call to the Swift cluster using the url given with the token as the X-Auth-Token header. * The Swift cluster makes an ReST call to the auth server to validate the - token for the given account hash, caching the result for future requests - up to the expiration the auth server returns. - * The auth server validates the token / account hash given and returns the - expiration for the token. + token, caching the result for future requests up to the expiration the + auth server returns. + * The auth server validates the token given and returns the expiration for + the token. * The Swift cluster completes the user's request. Another use case is creating a new account: @@ -103,17 +104,33 @@ class AuthController(object): Ring(os.path.join(self.swift_dir, 'account.ring.gz')) self.db_file = os.path.join(self.swift_dir, 'auth.db') self.conn = get_db_connection(self.db_file, okay_to_create=True) + try: + self.conn.execute('SELECT noaccess FROM account LIMIT 1') + except sqlite3.OperationalError, err: + if str(err) == 'no such column: noaccess': + self.conn.execute( + 'ALTER TABLE account ADD COLUMN noaccess TEXT') self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT)''') + user TEXT, password TEXT, noaccess TEXT)''') self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account ON account (account)''') + try: + self.conn.execute('SELECT user FROM token LIMIT 1') + except sqlite3.OperationalError, err: + if str(err) == 'no such column: user': + self.conn.execute('DROP INDEX IF EXISTS ix_token_created') + self.conn.execute('DROP INDEX IF EXISTS ix_token_cfaccount') + self.conn.execute('DROP TABLE IF EXISTS token') self.conn.execute('''CREATE TABLE IF NOT EXISTS token ( - cfaccount TEXT, token TEXT, created FLOAT)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount - ON token (cfaccount)''') + token TEXT, created FLOAT, + account TEXT, user TEXT, cfaccount TEXT)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token + ON token (token)''') self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created ON token (created)''') + self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account + ON token (account)''') self.conn.commit() def add_storage_account(self, account_name=''): @@ -202,38 +219,36 @@ class AuthController(object): (time() - self.token_life,)) conn.commit() - def validate_token(self, token, account_hash): + def validate_token(self, token): """ Tests if the given token is a valid token :param token: The token to validate - :param account_hash: The account hash the token is being used with - :returns: TTL if valid, False otherwise + :returns: (TTL, account, user, cfaccount) if valid, False otherwise """ begin = time() self.purge_old_tokens() rv = False with self.get_conn() as conn: row = conn.execute(''' - SELECT created FROM token - WHERE cfaccount = ? AND token = ?''', - (account_hash, token)).fetchone() + SELECT created, account, user, cfaccount FROM token + WHERE token = ?''', + (token,)).fetchone() if row is not None: created = row[0] if time() - created >= self.token_life: conn.execute(''' - DELETE FROM token - WHERE cfaccount = ? AND token = ?''', - (account_hash, token)) + DELETE FROM token WHERE token = ?''', (token,)) conn.commit() else: - rv = self.token_life - (time() - created) - self.logger.info('validate_token(%s, %s, _, _) = %s [%.02f]' % - (repr(token), repr(account_hash), repr(rv), - time() - begin)) + rv = (self.token_life - (time() - created), row[1], row[2], + row[3]) + self.logger.info('validate_token(%s, _, _) = %s [%.02f]' % + (repr(token), repr(rv), time() - begin)) return rv - def create_account(self, new_account, new_user, new_password): + def create_account(self, new_account, new_user, new_password, + noaccess=False): """ Handles the create_account call for developers, used to request an account be created both on a Swift cluster and in the auth server @@ -251,28 +266,51 @@ class AuthController(object): :param new_account: The name for the new account :param new_user: The name for the new user :param new_password: The password for the new account + :param noaccess: If true, the user will be granted no access to the + account by default; another user will have to add the + user to the ACLs for containers to grant access. - :returns: False if the create fails, storage url if successful + :returns: False if the create fails, 'already exists' if the user + already exists, or storage url if successful """ begin = time() if not all((new_account, new_user, new_password)): return False - account_hash = self.add_storage_account() - if not account_hash: - self.logger.info( - 'FAILED create_account(%s, %s, _,) [%.02f]' % - (repr(new_account), repr(new_user), time() - begin)) - return False - url = self.default_cluster_url.rstrip('/') + '/' + account_hash with self.get_conn() as conn: + row = conn.execute( + 'SELECT url FROM account WHERE account = ? AND user = ?', + (new_account, new_user)).fetchone() + if row: + self.logger.info( + 'ALREADY EXISTS create_account(%s, %s, _, %s) [%.02f]' % + (repr(new_account), repr(new_user), repr(noaccess), + time() - begin)) + return 'already exists' + row = conn.execute( + 'SELECT url, cfaccount FROM account WHERE account = ?', + (new_account,)).fetchone() + if row: + url = row[0] + account_hash = row[1] + else: + account_hash = self.add_storage_account() + if not account_hash: + self.logger.info( + 'FAILED create_account(%s, %s, _, %s) [%.02f]' % + (repr(new_account), repr(new_user), repr(noaccess), + time() - begin)) + return False + url = self.default_cluster_url.rstrip('/') + '/' + account_hash conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password) - VALUES (?, ?, ?, ?, ?)''', - (new_account, url, account_hash, new_user, new_password)) + (account, url, cfaccount, user, password, noaccess) + VALUES (?, ?, ?, ?, ?, ?)''', + (new_account, url, account_hash, new_user, new_password, + noaccess and 't' or '')) conn.commit() self.logger.info( - 'SUCCESS create_account(%s, %s, _) = %s [%.02f]' % - (repr(new_account), repr(new_user), repr(url), time() - begin)) + 'SUCCESS create_account(%s, %s, _, %s) = %s [%.02f]' % + (repr(new_account), repr(new_user), repr(noaccess), repr(url), + time() - begin)) return url def recreate_accounts(self): @@ -285,8 +323,8 @@ class AuthController(object): """ begin = time() with self.get_conn() as conn: - account_hashes = [r[0] for r in - conn.execute('SELECT cfaccount FROM account').fetchall()] + account_hashes = [r[0] for r in conn.execute( + 'SELECT distinct(cfaccount) FROM account').fetchall()] failures = [] for i, account_hash in enumerate(account_hashes): if not self.add_storage_account(account_hash): @@ -301,7 +339,7 @@ class AuthController(object): Hanles ReST request from Swift to validate tokens Valid URL paths: - * GET /token// + * GET /token/ If the HTTP equest returns with a 204, then the token is valid, and the TTL of the token will be available in the X-Auth-Ttl header. @@ -309,13 +347,14 @@ class AuthController(object): :param request: webob.Request object """ try: - _, account_hash, token = split_path(request.path, minsegs=3) + _, token = split_path(request.path, minsegs=2) except ValueError: return HTTPBadRequest() - ttl = self.validate_token(token, account_hash) - if not ttl: + validation = self.validate_token(token) + if not validation: return HTTPNotFound() - return HTTPNoContent(headers={'x-auth-ttl': ttl}) + return HTTPNoContent(headers={'X-Auth-TTL': validation[0], + 'X-Auth-User': ':'.join(validation[1:])}) def handle_account_create(self, request): """ @@ -339,7 +378,10 @@ class AuthController(object): if 'X-Auth-Key' not in request.headers: return HTTPBadRequest('X-Auth-Key is required') password = request.headers['x-auth-key'] - storage_url = self.create_account(account_name, user_name, password) + storage_url = self.create_account(account_name, user_name, password, + request.headers.get('x-user-no-access')) + if storage_url == 'already exists': + return HTTPBadRequest(storage_url) if not storage_url: return HTTPServiceUnavailable() return HTTPNoContent(headers={'x-storage-url': storage_url}) @@ -414,23 +456,25 @@ class AuthController(object): self.purge_old_tokens() with self.get_conn() as conn: row = conn.execute(''' - SELECT cfaccount, url FROM account + SELECT cfaccount, url, noaccess FROM account WHERE account = ? AND user = ? AND password = ?''', (account, user, password)).fetchone() if row is None: return HTTPUnauthorized() - cfaccount = row[0] + cfaccount = row[2] and '.none' or row[0] url = row[1] - row = conn.execute('SELECT token FROM token WHERE cfaccount = ?', - (cfaccount,)).fetchone() + row = conn.execute(''' + SELECT token FROM token WHERE account = ? AND user = ?''', + (account, user)).fetchone() if row: token = row[0] else: token = 'tk' + str(uuid4()) conn.execute(''' - INSERT INTO token (cfaccount, token, created) - VALUES (?, ?, ?)''', - (cfaccount, token, time())) + INSERT INTO token + (token, created, account, user, cfaccount) + VALUES (?, ?, ?, ?, ?)''', + (token, time(), account, user, cfaccount)) conn.commit() return HTTPNoContent(headers={'x-auth-token': token, 'x-storage-token': token, diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index eb920fad41..32800839ba 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -13,30 +13,135 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time +from time import time -from webob.request import Request -from webob.exc import HTTPUnauthorized, HTTPPreconditionFailed from eventlet.timeout import Timeout +from repoze.what.adapters import BaseSourceAdapter +from repoze.what.middleware import setup_auth +from repoze.what.predicates import in_any_group, NotAuthorizedError +from webob.exc import HTTPForbidden, HTTPUnauthorized -from swift.common.utils import split_path from swift.common.bufferedhttp import http_connect_raw as http_connect -from swift.common.utils import get_logger, cache_from_env -from swift.common.memcached import MemcacheRing +from swift.common.utils import cache_from_env, split_path -class DevAuthMiddleware(object): - """ - Auth Middleware that uses the dev auth server - """ +class DevAuthorization(object): - def __init__(self, app, conf, memcache_client=None, logger=None): + def __init__(self, app, conf): self.app = app - self.memcache_client = memcache_client - if logger is None: - self.logger = get_logger(conf) + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = self.clean_acl + return self.app(environ, start_response) + + def authorize(self, req): + version, account, container, obj = split_path(req.path, 1, 4, True) + if not account: + return self.denied_response(req) + groups = [account] + acl = self.parse_acl(getattr(req, 'acl', None)) + if acl: + referrers, accounts, users = acl + if referrers: + parts = req.referer.split('//', 1) + allow = False + if len(parts) == 2: + rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() + else: + rhost = 'unknown' + for mhost in referrers: + if mhost[0] == '-': + mhost = mhost[1:] + if mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = False + elif mhost == 'any' or mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = True + if allow: + return None + groups.extend(accounts) + groups.extend(users) + try: + in_any_group(*groups).check_authorization(req.environ) + except NotAuthorizedError: + return self.denied_response(req) + return None + + def denied_response(self, req): + if req.remote_user: + return HTTPForbidden(request=req) else: - self.logger = logger + return HTTPUnauthorized(request=req) + + def clean_acl(self, header_name, value): + values = [] + for raw_value in value.lower().split(','): + raw_value = raw_value.strip() + if raw_value: + if ':' in raw_value: + first, second = \ + (v.strip() for v in raw_value.split(':', 1)) + if not first: + raise ValueError('No value before colon in %s' % + repr(raw_value)) + if first == '.ref' and 'write' in header_name: + raise ValueError('Referrers not allowed in write ' + 'ACLs: %s' % repr(raw_value)) + if second: + if first == '.ref' and second[0] == '-': + second = second[1:].strip() + if not second: + raise ValueError('No value after referrer ' + 'deny designation in %s' % repr(raw_value)) + second = '-' + second + values.append('%s:%s' % (first, second)) + elif first == '.ref': + raise ValueError('No value after referrer designation ' + 'in %s' % repr(raw_value)) + else: + values.append(first) + else: + values.append(raw_value) + return ','.join(values) + + def parse_acl(self, acl_string): + if not acl_string: + return None + referrers = [] + accounts = [] + users = [] + for value in acl_string.split(','): + if value.startswith('.ref:'): + referrers.append(value[len('.ref:'):]) + elif ':' in value: + users.append(value) + else: + accounts.append(value) + return (referrers, accounts, users) + + +class DevIdentifier(object): + + def __init__(self, conf): + self.conf = conf + + def identify(self, env): + return {'token': + env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))} + + def remember(self, env, identity): + return [] + + def forget(self, env, identity): + return [] + + +class DevAuthenticator(object): + + def __init__(self, conf): self.conf = conf self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) @@ -44,69 +149,135 @@ class DevAuthMiddleware(object): conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes') self.timeout = int(conf.get('node_timeout', 10)) - def __call__(self, env, start_response): - if self.memcache_client is None: - self.memcache_client = cache_from_env(env) - req = Request(env) - if 'x-storage-token' in req.headers and \ - 'x-auth-token' not in req.headers: - req.headers['x-auth-token'] = req.headers['x-storage-token'] - try: - version, account, container, obj = split_path(req.path, 1, 4, True) - except ValueError, e: - version = account = container = obj = None - if account is None: - return HTTPPreconditionFailed(request=req, body='Bad URL')( - env, start_response) - if not req.headers.get('x-auth-token'): - return HTTPPreconditionFailed(request=req, - body='Missing Auth Token')(env, start_response) - if not self.auth(account, req.headers['x-auth-token']): - return HTTPUnauthorized(request=req)(env, start_response) - - # If we get here, then things should be good. - return self.app(env, start_response) - - def auth(self, account, token): - """ - Dev authorization implmentation - - :param account: account name - :param token: auth token - - :returns: True if authorization is successful, False otherwise - """ - key = 'auth/%s/%s' % (account, token) - now = time.time() - cached_auth_data = self.memcache_client.get(key) + def authenticate(self, env, identity): + token = identity.get('token') + if not token: + return None + memcache_client = cache_from_env(env) + key = 'devauth/%s' % token + cached_auth_data = memcache_client.get(key) if cached_auth_data: - start, expiration = cached_auth_data - if now - start <= expiration: - return True - try: - with Timeout(self.timeout): - conn = http_connect(self.auth_host, self.auth_port, 'GET', - '/token/%s/%s' % (account, token), ssl=self.ssl) - resp = conn.getresponse() - resp.read() - conn.close() - if resp.status == 204: - validated = float(resp.getheader('x-auth-ttl')) - else: - validated = False - except: - self.logger.exception('ERROR with auth') - return False - if not validated: - return False - else: - val = (now, validated) - self.memcache_client.set(key, val, timeout=validated) - return True + start, expiration, user = cached_auth_data + if time() - start <= expiration: + return user + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s' % token, ssl=self.ssl) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status == 204: + expiration = float(resp.getheader('x-auth-ttl')) + user = resp.getheader('x-auth-user') + memcache_client.set(key, (time(), expiration, user), + timeout=expiration) + return user + return None + + +class DevChallenger(object): + + def __init__(self, conf): + self.conf = conf + + def challenge(self, env, status, app_headers, forget_headers): + def no_challenge(env, start_response): + start_response(str(status), []) + return [] + return no_challenge + + +class DevGroupSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevGroupSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, credentials): + creds = credentials['repoze.what.userid'].split(':') + if len(creds) != 3: + return set() + rv = set([creds[0], ':'.join(creds[:2]), creds[2]]) + return rv + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + + +class DevPermissionSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, group_name): + return set([n for (n, p) in self.sections.items() + if group_name in p]) + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def auth_filter(app): - return DevAuthMiddleware(app, conf) + return setup_auth(DevAuthorization(app, conf), + group_adapters={'all_groups': DevGroupSourceAdapter()}, + permission_adapters={'all_perms': DevPermissionSourceAdapter()}, + identifiers=[('devauth', DevIdentifier(conf))], + authenticators=[('devauth', DevAuthenticator(conf))], + challengers=[('devauth', DevChallenger(conf))]) return auth_filter diff --git a/swift/container/server.py b/swift/container/server.py index 23a9d0b71c..63a44fcabc 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -44,6 +44,9 @@ DATADIR = 'containers' class ContainerController(object): """WSGI Controller for the container server.""" + # Ensure these are all lowercase + save_headers = ['x-container-read', 'x-container-write'] + def __init__(self, conf): self.logger = get_logger(conf) self.root = conf.get('devices', '/srv/node/') @@ -192,7 +195,8 @@ class ContainerController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-container-meta-')) + if key.lower() in self.save_headers or + key.lower().startswith('x-container-meta-')) if metadata: broker.update_metadata(metadata) resp = self.account_update(req, account, container, broker) @@ -373,7 +377,8 @@ class ContainerController(object): metadata = {} metadata.update((key, (value, timestamp)) for key, value in req.headers.iteritems() - if key.lower().startswith('x-container-meta-')) + if key.lower() in self.save_headers or + key.lower().startswith('x-container-meta-')) if metadata: broker.update_metadata(metadata) return HTTPNoContent(request=req) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 140bc5dbc4..aea7b618bd 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -17,6 +17,7 @@ from __future__ import with_statement import mimetypes import os import time +import traceback from ConfigParser import ConfigParser from urllib import unquote, quote import uuid @@ -73,6 +74,22 @@ def public(func): return wrapped +def delay_denial(func): + """ + Decorator to declare which methods should have any swift.authorize call + delayed. This is so the method can load the Request object up with + additional information that may be needed by the authorization system. + + :param func: function to delay authorization on + """ + func.delay_denial = True + + @functools.wraps(func) + def wrapped(*a, **kw): + return func(*a, **kw) + return wrapped + + class Controller(object): """Base WSGI controller class for the proxy""" @@ -206,19 +223,28 @@ class Controller(object): :param account: account name for the container :param container: container name to look up - :returns: tuple of (container partition, container nodes) or - (None, None) if the container does not exist + :returns: tuple of (container partition, container nodes, container + read acl, container write acl) or (None, None, None, None) if + the container does not exist """ partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) cache_key = 'container%s' % path + # Older memcache values (should be treated as if they aren't there): # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses - if self.app.memcache.get(cache_key) == 200: - return partition, nodes + # Newer memcache values: + # [older status value from above, read acl, write acl] + cache_value = self.app.memcache.get(cache_key) + if hasattr(cache_value, '__iter__'): + status, read_acl, write_acl = cache_value + if status == 200: + return partition, nodes, read_acl, write_acl if not self.account_info(account)[1]: - return (None, None) + return (None, None, None, None) result_code = 0 + read_acl = None + write_acl = None attempts_left = self.app.container_ring.replica_count headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.container_ring): @@ -233,6 +259,8 @@ class Controller(object): body = resp.read() if 200 <= resp.status <= 299: result_code = 200 + read_acl = resp.getheader('x-container-read') + write_acl = resp.getheader('x-container-write') break elif resp.status == 404: result_code = 404 if not result_code else -1 @@ -251,10 +279,11 @@ class Controller(object): cache_timeout = self.app.recheck_container_existence else: cache_timeout = self.app.recheck_container_existence * 0.1 - self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) + self.app.memcache.set(cache_key, (result_code, read_acl, write_acl), + timeout=cache_timeout) if result_code == 200: - return partition, nodes - return (None, None) + return partition, nodes, read_acl, write_acl + return (None, None, None, None) def iter_nodes(self, partition, nodes, ring): """ @@ -474,6 +503,12 @@ class ObjectController(Controller): def GETorHEAD(self, req): """Handle HTTP GET or HEAD requests.""" + if 'swift.authorize' in req.environ: + req.acl = \ + self.container_info(self.account_name, self.container_name)[2] + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp partition, nodes = self.app.object_ring.get_nodes( self.account_name, self.container_name, self.object_name) return self.GETorHEAD_base(req, 'Object', partition, @@ -481,13 +516,30 @@ class ObjectController(Controller): req.path_info, self.app.object_ring.replica_count) @public + @delay_denial + def GET(self, req): + """Handler for HTTP GET requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial + def HEAD(self, req): + """Handler for HTTP HEAD requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial def POST(self, req): """HTTP POST request handler.""" error_response = check_metadata(req, 'object') if error_response: return error_response - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -521,10 +573,15 @@ class ObjectController(Controller): bodies, 'Object POST') @public + @delay_denial def PUT(self, req): """HTTP PUT request handler.""" - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -701,10 +758,15 @@ class ObjectController(Controller): return resp @public + @delay_denial def DELETE(self, req): """HTTP DELETE request handler.""" - container_partition, containers = \ + container_partition, containers, _, req.acl = \ self.container_info(self.account_name, self.container_name) + if 'swift.authorize' in req.environ: + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp if not containers: return HTTPNotFound(request=req) containers = self.get_update_nodes(container_partition, containers, @@ -771,11 +833,25 @@ class ObjectController(Controller): class ContainerController(Controller): """WSGI controller for container requests""" + # Ensure these are all lowercase + pass_through_headers = ['x-container-read', 'x-container-write'] + def __init__(self, app, account_name, container_name, **kwargs): Controller.__init__(self, app) self.account_name = unquote(account_name) self.container_name = unquote(container_name) + def clean_acls(self, req): + if 'swift.clean_acl' in req.environ: + for header in ('x-container-read', 'x-container-write'): + if header in req.headers: + try: + req.headers[header] = \ + req.environ['swift.clean_acl'](header, + req.headers[header]) + except ValueError, err: + return HTTPBadRequest(request=req, body=str(err)) + def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" if not self.account_info(self.account_name)[1]: @@ -784,8 +860,25 @@ class ContainerController(Controller): self.account_name, self.container_name) resp = self.GETorHEAD_base(req, 'Container', part, nodes, req.path_info, self.app.container_ring.replica_count) + if 'swift.authorize' in req.environ: + req.acl = resp.headers.get('x-container-read') + aresp = req.environ['swift.authorize'](req) + if aresp: + return aresp return resp + @public + @delay_denial + def GET(self, req): + """Handler for HTTP GET requests.""" + return self.GETorHEAD(req) + + @public + @delay_denial + def HEAD(self, req): + """Handler for HTTP HEAD requests.""" + return self.GETorHEAD(req) + @public def PUT(self, req): """HTTP PUT request handler.""" @@ -806,8 +899,10 @@ class ContainerController(Controller): self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} + self.clean_acls(req) headers.update(value for value in req.headers.iteritems() - if value[0].lower().startswith('x-container-meta-')) + if value[0].lower() in self.pass_through_headers or + value[0].lower().startswith('x-container-meta-')) statuses = [] reasons = [] bodies = [] @@ -863,8 +958,10 @@ class ContainerController(Controller): self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} + self.clean_acls(req) headers.update(value for value in req.headers.iteritems() - if value[0].lower().startswith('x-container-meta-')) + if value[0].lower() in self.pass_through_headers or + value[0].lower().startswith('x-container-meta-')) statuses = [] reasons = [] bodies = [] @@ -1118,7 +1215,8 @@ class BaseApplication(object): self.posthooklogger(env, req) return response except: - print "EXCEPTION IN __call__: %s" % env + print "EXCEPTION IN __call__: %s: %s" % \ + (traceback.format_exc(), env) start_response('500 Server Error', [('Content-Type', 'text/plain')]) return ['Internal server error.\n'] @@ -1160,12 +1258,28 @@ class BaseApplication(object): controller.trans_id = req.headers.get('x-cf-trans-id', '-') try: handler = getattr(controller, req.method) - if getattr(handler, 'publicly_accessible'): - if path_parts['version']: - req.path_info_pop() - return handler(req) + if not getattr(handler, 'publicly_accessible'): + handler = None except AttributeError: + handler = None + if not handler: return HTTPMethodNotAllowed(request=req) + if path_parts['version']: + req.path_info_pop() + if 'swift.authorize' in req.environ: + # We call authorize before the handler, always. If authorized, + # we remove the swift.authorize hook so isn't ever called + # again. If not authorized, we return the denial unless the + # controller's method indicates it'd like to gather more + # information and try again later. + resp = req.environ['swift.authorize'](req) + if resp: + if not getattr(handler, 'delay_denial', None) and \ + 'swift.authorize' in req.environ: + return resp + else: + del req.environ['swift.authorize'] + return handler(req) except Exception: self.logger.exception('ERROR Unhandled exception in request') return HTTPServerError(request=req) @@ -1187,7 +1301,9 @@ class Application(BaseApplication): return req.response def posthooklogger(self, env, req): - response = req.response + response = getattr(req, 'response', None) + if not response: + return trans_time = '%.4f' % (time.time() - req.start_time) the_request = quote(unquote(req.path)) if req.query_string: @@ -1215,7 +1331,8 @@ class Application(BaseApplication): status_int, req.referer or '-', req.user_agent or '-', - req.headers.get('x-auth-token', '-'), + '%s:%s' % (req.remote_user or '', + req.headers.get('x-auth-token', '-')), getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), diff --git a/test/functional/tests.py b/test/functional/tests.py index 3a1f35fc36..ae396f3fb0 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -106,9 +106,12 @@ class Base(unittest.TestCase): self.assert_(response_body == body, 'Body returned: %s' % (response_body)) - def assert_status(self, status): - self.assert_(self.env.conn.response.status == status, - 'Status returned: %d' % (self.env.conn.response.status)) + def assert_status(self, status_or_statuses): + self.assert_(self.env.conn.response.status == status_or_statuses or + (hasattr(status_or_statuses, '__iter__') and + self.env.conn.response.status in status_or_statuses), + 'Status returned: %d Expected: %s' % + (self.env.conn.response.status, status_or_statuses)) class Base2(object): def setUp(self): @@ -148,11 +151,11 @@ class TestAccount(Base): def testNoAuthToken(self): self.assertRaises(ResponseError, self.env.account.info, cfg={'no_auth_token':True}) - self.assert_status(412) + self.assert_status([401, 412]) self.assertRaises(ResponseError, self.env.account.containers, cfg={'no_auth_token':True}) - self.assert_status(412) + self.assert_status([401, 412]) def testInvalidUTF8Path(self): invalid_utf8 = Utils.create_utf8_name()[::-1] @@ -1123,7 +1126,8 @@ class TestFile(Base): self.assert_status(400) # bad request types - for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'): + #for req in ('LICK', 'GETorHEAD_base', 'container_info', 'best_response'): + for req in ('LICK', 'GETorHEAD_base'): self.env.account.conn.make_request(req) self.assert_status(405) diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py index 1c805f854d..8bd46b462b 100644 --- a/test/functionalnosetests/swift_testing.py +++ b/test/functionalnosetests/swift_testing.py @@ -10,11 +10,11 @@ from swift.common.client import get_auth, http_connection swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') -swift_test_user = os.environ.get('SWIFT_TEST_USER') -swift_test_key = os.environ.get('SWIFT_TEST_KEY') +swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] +swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] # If no environment set, fall back to old school conf file -if not all([swift_test_auth, swift_test_user, swift_test_key]): +if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]): conf = ConfigParser() class Sectionizer(object): def __init__(self, fp): @@ -32,16 +32,36 @@ if not all([swift_test_auth, swift_test_user, swift_test_key]): if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): swift_test_auth = 'https' swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf - swift_test_user = '%(account)s:%(username)s' % conf - swift_test_key = conf['password'] + swift_test_user[0] = '%(account)s:%(username)s' % conf + swift_test_key[0] = conf['password'] + try: + swift_test_user[1] = '%(account2)s:%(username2)s' % conf + swift_test_key[1] = conf['password2'] + except KeyError, err: + pass # old conf, no second account tests can be run + try: + swift_test_user[2] = '%(account)s:%(username3)s' % conf + swift_test_key[2] = conf['password3'] + except KeyError, err: + pass # old conf, no third account tests can be run except IOError, err: if err.errno != errno.ENOENT: raise -skip = not all([swift_test_auth, swift_test_user, swift_test_key]) +skip = not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]) if skip: print >>sys.stderr, 'SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG' +skip2 = not all([not skip, swift_test_user[1], swift_test_key[1]]) +if not skip and skip2: + print >>sys.stderr, \ + 'SKIPPING SECOND ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + +skip3 = not all([not skip, swift_test_user[2], swift_test_key[2]]) +if not skip and skip3: + print >>sys.stderr, \ + 'SKIPPING THIRD ACCOUNT FUNCTIONAL TESTS DUE TO NO CONFIG FOR THEM' + class AuthError(Exception): pass @@ -51,29 +71,44 @@ class InternalServerError(Exception): pass -url = token = parsed = conn = None +url = [None, None, None] +token = [None, None, None] +parsed = [None, None, None] +conn = [None, None, None] def retry(func, *args, **kwargs): + """ + You can use the kwargs to override the 'retries' (default: 5) and + 'use_account' (default: 1). + """ global url, token, parsed, conn retries = kwargs.get('retries', 5) + use_account = 1 + if 'use_account' in kwargs: + use_account = kwargs['use_account'] + del kwargs['use_account'] + use_account -= 1 attempts = 0 backoff = 1 while attempts <= retries: attempts += 1 try: - if not url or not token: - url, token = \ - get_auth(swift_test_auth, swift_test_user, swift_test_key) - parsed = conn = None - if not parsed or not conn: - parsed, conn = http_connection(url) - return func(url, token, parsed, conn, *args, **kwargs) + if not url[use_account] or not token[use_account]: + url[use_account], token[use_account] = \ + get_auth(swift_test_auth, swift_test_user[use_account], + swift_test_key[use_account]) + parsed[use_account] = conn[use_account] = None + if not parsed[use_account] or not conn[use_account]: + parsed[use_account], conn[use_account] = \ + http_connection(url[use_account]) + return func(url[use_account], token[use_account], + parsed[use_account], conn[use_account], *args, **kwargs) except (socket.error, HTTPException): if attempts > retries: raise - parsed = conn = None + parsed[use_account] = conn[use_account] = None except AuthError, err: - url = token = None + url[use_account] = token[use_account] = None continue except InternalServerError, err: pass diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py index 9c36b460b8..15bee7fbdb 100755 --- a/test/functionalnosetests/test_container.py +++ b/test/functionalnosetests/test_container.py @@ -1,12 +1,14 @@ #!/usr/bin/python +import json import unittest from uuid import uuid4 from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH -from swift_testing import check_response, retry, skip +from swift_testing import check_response, retry, skip, skip2, skip3, \ + swift_test_user class TestContainer(unittest.TestCase): @@ -26,6 +28,26 @@ class TestContainer(unittest.TestCase): def tearDown(self): if skip: return + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name + '?format=json', + '', {'X-Auth-Token': token}) + return check_response(conn) + def delete(url, token, parsed, conn, obj): + conn.request('DELETE', + '/'.join([parsed.path, self.name, obj['name']]), '', + {'X-Auth-Token': token}) + return check_response(conn) + while True: + resp = retry(get) + body = resp.read() + self.assert_(resp.status // 100 == 2, resp.status) + objs = json.loads(body) + if not objs: + break + for obj in objs: + resp = retry(delete, obj) + resp.read() + self.assertEquals(resp.status, 204) def delete(url, token, parsed, conn): conn.request('DELETE', parsed.path + '/' + self.name, '', {'X-Auth-Token': token}) @@ -297,6 +319,206 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEquals(resp.status, 400) + def test_public_container(self): + if skip: + return + def get(url, token, parsed, conn): + conn.request('GET', parsed.path + '/' + self.name) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.ref:any'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 204) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after '), err) + + def test_cross_account_container(self): + if skip or skip2: + return + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': 'test2', + 'X-Container-Write': 'test2'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now use the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # Make the container private again + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': '', + 'X-Container-Write': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can't access the container with the second account again + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + + def test_cross_account_public_container(self): + if skip or skip2: + return + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the second account + def get2(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container completely public + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.ref:any'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the second account + def put2(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the second account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': 'test2'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the second account + resp = retry(get2, use_account=2) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the second account + resp = retry(put2, use_account=2) + resp.read() + self.assertEquals(resp.status, 201) + + def test_noaccess_user(self): + if skip or skip3: + return + # Obtain the first account's string + first_account = ['unknown'] + def get1(url, token, parsed, conn): + first_account[0] = parsed.path + conn.request('HEAD', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get1) + resp.read() + # Ensure we can't access the container with the third account + def get3(url, token, parsed, conn): + conn.request('GET', first_account[0] + '/' + self.name, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Make the container accessible by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can now read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # But we shouldn't be able to write with the third account + def put3(url, token, parsed, conn): + conn.request('PUT', first_account[0] + '/' + self.name + '/object', + 'test object', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 403) + # Now make the container also writeable by the third account + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, + 'X-Container-Write': swift_test_user[2]}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + # Ensure we can still read the container with the third account + resp = retry(get3, use_account=3) + resp.read() + self.assertEquals(resp.status, 204) + # And that we can now write with the third account + resp = retry(put3, use_account=3) + resp.read() + self.assertEquals(resp.status, 201) + if __name__ == '__main__': unittest.main() diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py new file mode 100644 index 0000000000..256068d766 --- /dev/null +++ b/test/functionalnosetests/test_object.py @@ -0,0 +1,90 @@ +#!/usr/bin/python + +import unittest +from uuid import uuid4 + +from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ + MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH + +from swift_testing import check_response, retry, skip + + +class TestObject(unittest.TestCase): + + def setUp(self): + if skip: + return + self.container = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + self.obj = uuid4().hex + def put(url, token, parsed, conn): + conn.request('PUT', '%s/%s/%s' % (parsed.path, self.container, + self.obj), 'test', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(put) + resp.read() + self.assertEquals(resp.status, 201) + + def tearDown(self): + if skip: + return + def delete(url, token, parsed, conn): + conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container, + self.obj), '', {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + def delete(url, token, parsed, conn): + conn.request('DELETE', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token}) + return check_response(conn) + resp = retry(delete) + resp.read() + self.assertEquals(resp.status, 204) + + def test_public_object(self): + if skip: + return + def get(url, token, parsed, conn): + conn.request('GET', + '%s/%s/%s' % (parsed.path, self.container, self.obj)) + return check_response(conn) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, + 'X-Container-Read': '.ref:any'}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + resp = retry(get) + resp.read() + self.assertEquals(resp.status, 200) + def post(url, token, parsed, conn): + conn.request('POST', parsed.path + '/' + self.container, '', + {'X-Auth-Token': token, 'X-Container-Read': ''}) + return check_response(conn) + resp = retry(post) + resp.read() + self.assertEquals(resp.status, 204) + try: + resp = retry(get) + raise Exception('Should not have been able to GET') + except Exception, err: + self.assert_(str(err).startswith('No result after ')) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index 31495401f2..cdaffc33e5 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -113,20 +113,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - self.assertEquals(self.controller.validate_token(token + 'bad', - cfaccount), False) - - def test_validate_token_non_existant_cfaccount(self): - auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - self.assertEquals(self.controller.validate_token(token, - cfaccount + 'bad'), False) + self.assertEquals(self.controller.validate_token(token + 'bad'), False) def test_validate_token_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) @@ -137,7 +124,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_validate_token_expired(self): @@ -152,12 +139,10 @@ class TestAuthServer(unittest.TestCase): headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token( - token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) auth_server.time = lambda: 1 + self.controller.token_life - self.assertEquals(self.controller.validate_token( - token, cfaccount), False) + self.assertEquals(self.controller.validate_token(token), False) finally: auth_server.time = orig_time @@ -244,7 +229,7 @@ class TestAuthServer(unittest.TestCase): rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '4', repr(rv)) failed = rv.split('[', 1)[-1][:-1].split(', ') - self.assertEquals(failed, [repr(a) for a in cfaccounts]) + self.assertEquals(set(failed), set(repr(a) for a in cfaccounts)) def test_recreate_accounts_several_fail_some(self): auth_server.http_connect = fake_http_connect(201, 201, 201) @@ -266,11 +251,8 @@ class TestAuthServer(unittest.TestCase): rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '4', repr(rv)) failed = rv.split('[', 1)[-1][:-1].split(', ') - expected = [] - for i, value in enumerate(cfaccounts): - if not i % 2: - expected.append(repr(value)) - self.assertEquals(failed, expected) + self.assertEquals( + len(set(repr(a) for a in cfaccounts) - set(failed)), 2) def test_auth_bad_path(self): self.assertRaises(ValueError, self.controller.handle_auth, @@ -349,7 +331,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Storage-User': 'tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_SOSO_good_Mosso_headers(self): @@ -361,7 +343,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_SOSO_bad_Mosso_headers(self): @@ -469,7 +451,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Auth-User': 'test:tester', 'X-Auth-Key': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_auth_Mosso_good_SOSO_header_names(self): @@ -481,7 +463,7 @@ class TestAuthServer(unittest.TestCase): headers={'X-Storage-User': 'test:tester', 'X-Storage-Pass': 'testing'})) token = res.headers['x-storage-token'] - ttl = self.controller.validate_token(token, cfaccount) + ttl = self.controller.validate_token(token) self.assert_(ttl > 0, repr(ttl)) def test_basic_logging(self): @@ -493,8 +475,8 @@ class TestAuthServer(unittest.TestCase): auth_server.http_connect = fake_http_connect(201, 201, 201) url = self.controller.create_account('test', 'tester', 'testing') self.assertEquals(log.getvalue().rsplit(' ', 1)[0], - "auth SUCCESS create_account('test', 'tester', _) = %s" % - repr(url)) + "auth SUCCESS create_account('test', 'tester', _, False) = %s" + % repr(url)) log.truncate(0) def start_response(*args): pass diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index cd5a02c91d..08f8898d1b 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -100,77 +100,58 @@ def start_response(*args): pass class TestAuth(unittest.TestCase): + # TODO: With the auth refactor, these tests have to be refactored as well. + # I brought some over from another refactor I've been trying, but these + # also need work. - def setUp(self): - self.test_auth = auth.DevAuthMiddleware( - FakeApp(), {}, FakeMemcache(), Logger()) + def test_clean_acl(self): + devauth = auth.DevAuthorization(None, None) + value = devauth.clean_acl('header', '.ref:any') + self.assertEquals(value, '.ref:any') + value = devauth.clean_acl('header', '.ref:specific.host') + self.assertEquals(value, '.ref:specific.host') + value = devauth.clean_acl('header', '.ref:.ending.with') + self.assertEquals(value, '.ref:.ending.with') + value = devauth.clean_acl('header', '.ref:one,.ref:two') + self.assertEquals(value, '.ref:one,.ref:two') + value = devauth.clean_acl('header', '.ref:any,.ref:-specific.host') + self.assertEquals(value, '.ref:any,.ref:-specific.host') + value = devauth.clean_acl('header', '.ref:any,.ref:-.ending.with') + self.assertEquals(value, '.ref:any,.ref:-.ending.with') + value = devauth.clean_acl('header', '.ref:one,.ref:-two') + self.assertEquals(value, '.ref:one,.ref:-two') + value = devauth.clean_acl('header', + ' .ref : one , ,, .ref:two , .ref : - three ') + self.assertEquals(value, '.ref:one,.ref:two,.ref:-three') + self.assertRaises(ValueError, devauth.clean_acl, 'header', '.ref:') + self.assertRaises(ValueError, devauth.clean_acl, 'header', ' .ref : ') + self.assertRaises(ValueError, devauth.clean_acl, 'header', + 'user , .ref : ') + self.assertRaises(ValueError, devauth.clean_acl, 'header', '.ref:-') + self.assertRaises(ValueError, devauth.clean_acl, 'header', + ' .ref : - ') + self.assertRaises(ValueError, devauth.clean_acl, 'header', + 'user , .ref : - ') - def test_auth_fail(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(404) - self.assertFalse(self.test_auth.auth('a','t')) - finally: - auth.http_connect = old_http_connect - - def test_auth_success(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - self.assertTrue(self.test_auth.auth('a','t')) - finally: - auth.http_connect = old_http_connect - - def test_auth_memcache(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - self.assertTrue(self.test_auth.auth('a','t')) - auth.http_connect = mock_http_connect(404) - # Should still be in memcache - self.assertTrue(self.test_auth.auth('a','t')) - finally: - auth.http_connect = old_http_connect - - def test_middleware_success(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v/a/c/o', headers={'x-auth-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, 'OK') - finally: - auth.http_connect = old_http_connect - - def test_middleware_no_header(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v/a/c/o') - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, ['Missing Auth Token']) - finally: - auth.http_connect = old_http_connect - - def test_middleware_storage_token(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v/a/c/o', headers={'x-storage-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, 'OK') - finally: - auth.http_connect = old_http_connect - - def test_middleware_only_version(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, {'x-auth-ttl':'1234'}) - req = Request.blank('/v', headers={'x-auth-token':'t'}) - resp = self.test_auth(req.environ, start_response) - self.assertEquals(resp, ['Bad URL']) - finally: - auth.http_connect = old_http_connect + def test_parse_acl(self): + devauth = auth.DevAuthorization(None, None) + self.assertEquals(devauth.parse_acl(None), None) + self.assertEquals(devauth.parse_acl(''), None) + self.assertEquals(devauth.parse_acl('.ref:ref1'), + (['ref1'], [], [])) + self.assertEquals(devauth.parse_acl('.ref:-ref1'), + (['-ref1'], [], [])) + self.assertEquals(devauth.parse_acl('account:user'), + ([], [], ['account:user'])) + self.assertEquals(devauth.parse_acl('account'), + ([], ['account'], [])) + self.assertEquals( + devauth.parse_acl('acc1,acc2:usr2,.ref:ref3,.ref:-ref4'), + (['ref3', '-ref4'], ['acc1'], ['acc2:usr2'])) + self.assertEquals(devauth.parse_acl( + 'acc1,acc2:usr2,.ref:ref3,acc3,acc4:usr4,.ref:ref5,.ref:-ref6'), + (['ref3', 'ref5', '-ref6'], ['acc1', 'acc3'], + ['acc2:usr2', 'acc4:usr4'])) if __name__ == '__main__': diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 639bc4ec5c..dfa2ef2d57 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -55,6 +55,51 @@ class TestContainerController(unittest.TestCase): """ Tear down for testing swift.object_server.ObjectController """ rmtree(self.testdir, ignore_errors=1) + def test_acl_container(self): + # Ensure no acl by default + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '0'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assert_('x-container-read' not in response.headers) + self.assert_('x-container-write' not in response.headers) + # Ensure POSTing acls works + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': '1', 'X-Container-Read': '.ref:any', + 'X-Container-Write': 'account:user'}) + self.controller.POST(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assertEquals(response.headers.get('x-container-read'), + '.ref:any') + self.assertEquals(response.headers.get('x-container-write'), + 'account:user') + # Ensure we can clear acls on POST + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Timestamp': '3', 'X-Container-Read': '', + 'X-Container-Write': ''}) + self.controller.POST(req) + req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assert_('x-container-read' not in response.headers) + self.assert_('x-container-write' not in response.headers) + # Ensure PUTing acls works + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Timestamp': '4', 'X-Container-Read': '.ref:any', + 'X-Container-Write': 'account:user'}) + self.controller.PUT(req) + req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'}) + response = self.controller.HEAD(req) + self.assert_(response.status.startswith('204')) + self.assertEquals(response.headers.get('x-container-read'), + '.ref:any') + self.assertEquals(response.headers.get('x-container-write'), + 'account:user') + def test_HEAD(self): req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', 'HTTP_X_TIMESTAMP': '0'}) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index ab56dc740a..ce8d1ee4cb 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -203,7 +203,7 @@ class TestProxyServer(unittest.TestCase): app = MyApp(None, FakeMemcache(), account_ring=FakeRing(), container_ring=FakeRing(), object_ring=FakeRing()) req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + app.update_request(req) resp = app.handle_request(req) self.assertEquals(resp.status_int, 500) @@ -224,14 +224,14 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c/o', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -244,7 +244,7 @@ class TestObjectController(unittest.TestCase): give_content_type=lambda content_type: self.assertEquals(content_type, expected.next())) req = Request.blank('/a/c/%s' % filename, {}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) test_content_type('test.jpg', iter(['', '', '', 'image/jpeg', 'image/jpeg', 'image/jpeg'])) @@ -261,7 +261,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) self.app.memcache.store = {} res = controller.PUT(req) expected = str(expected) @@ -296,7 +296,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -330,7 +330,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} proxy_server.http_connect = mock_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) - req.account = 'a' + self.app.update_request(req) req.body_file = StringIO('some data') res = controller.PUT(req) expected = str(expected) @@ -347,7 +347,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Length': str(MAX_FILE_SIZE + 1), 'Content-Type': 'foo/bar'}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 413) @@ -379,7 +379,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = mock_http_connect(*statuses) req = Request.blank('/a/c/o.jpg', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(str(expected))], @@ -397,7 +397,7 @@ class TestObjectController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -417,7 +417,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/a/c/o', {}) - req.account = 'a' + self.app.update_request(req) res = controller.DELETE(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -436,7 +436,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses) self.app.memcache.store = {} req = Request.blank('/a/c/o', {}) - req.account = 'a' + self.app.update_request(req) res = controller.HEAD(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -460,14 +460,14 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x'*256}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 202) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', 'X-Object-Meta-Foo': 'x'*257}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -481,14 +481,14 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', ('X-Object-Meta-'+'x'*128): 'x'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 202) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers={ 'Content-Type': 'foo/bar', ('X-Object-Meta-'+'x'*129): 'x'}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -500,7 +500,7 @@ class TestObjectController(unittest.TestCase): headers.update({'Content-Type': 'foo/bar'}) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers=headers) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -512,7 +512,7 @@ class TestObjectController(unittest.TestCase): headers.update({'Content-Type': 'foo/bar'}) proxy_server.http_connect = fake_http_connect(202, 202, 202) req = Request.blank('/a/c/o', {}, headers=headers) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(res.status_int, 400) @@ -542,7 +542,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -554,7 +554,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(201, 201, 201) # obj obj obj @@ -583,7 +583,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT', 'wsgi.input': SlowBody()}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -607,7 +607,7 @@ class TestObjectController(unittest.TestCase): dev['ip'] = '127.0.0.1' dev['port'] = 1 req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -649,7 +649,7 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = \ @@ -663,7 +663,7 @@ class TestObjectController(unittest.TestCase): environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '4', 'Content-Type': 'text/plain'}, body=' ') - req.account = 'account' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 503) @@ -708,7 +708,7 @@ class TestObjectController(unittest.TestCase): def test_proxy_passes_content_type(self): with save_globals(): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'GET'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = fake_http_connect(200, 200, 200) @@ -728,7 +728,7 @@ class TestObjectController(unittest.TestCase): def test_proxy_passes_content_length_on_head(self): with save_globals(): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') proxy_server.http_connect = fake_http_connect(200, 200, 200) @@ -777,7 +777,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'DELETE'}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, 'DELETE')(req) self.assertEquals(resp.status_int, 200) @@ -853,7 +853,7 @@ class TestObjectController(unittest.TestCase): proxy_server.http_connect = \ fake_http_connect(404, 404, 404, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 404) @@ -861,7 +861,7 @@ class TestObjectController(unittest.TestCase): fake_http_connect(404, 404, 404, 200, 200, 200) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, headers={'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 404) @@ -874,7 +874,7 @@ class TestObjectController(unittest.TestCase): # acct cont obj obj obj req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) @@ -883,7 +883,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * MAX_META_NAME_LENGTH) : 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -891,7 +891,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)) : 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -900,7 +900,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -908,7 +908,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Object-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -918,7 +918,7 @@ class TestObjectController(unittest.TestCase): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -927,7 +927,7 @@ class TestObjectController(unittest.TestCase): headers['X-Object-Meta-%d' % x] = 'v' req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -946,7 +946,7 @@ class TestObjectController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -954,7 +954,7 @@ class TestObjectController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.PUT(req) self.assertEquals(resp.status_int, 400) @@ -964,7 +964,7 @@ class TestObjectController(unittest.TestCase): 'container', 'object') req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 201, 201, 201) # acct cont obj obj obj @@ -974,7 +974,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': 'c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) # acct cont acct cont objc obj obj obj @@ -986,7 +986,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 200, 200, 201, 201, 201) # acct cont acct cont objc obj obj obj @@ -998,7 +998,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 503, 503, 503) # acct cont objc objc objc @@ -1009,7 +1009,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 404, 404, 404) # acct cont objc objc objc @@ -1020,7 +1020,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0', 'X-Copy-From': '/c/o'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 404, 404, 200, 201, 201, 201) # acct cont objc objc objc obj obj obj @@ -1032,7 +1032,7 @@ class TestObjectController(unittest.TestCase): headers={'Content-Length': '0', 'X-Copy-From': '/c/o', 'X-Object-Meta-Ours': 'okay'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = \ fake_http_connect(200, 200, 200, 201, 201, 201) # acct cont objc obj obj obj @@ -1434,7 +1434,7 @@ class TestObjectController(unittest.TestCase): 'container', 'object') req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '0'}) - req.account = 'a' + self.app.update_request(req) proxy_server.http_connect = fake_http_connect(200, 201, 201, 201, etags=[None, '68b329da9893e34099c7d8ad5cb9c940', @@ -1452,7 +1452,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '10'}, body='1234567890') - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assert_(hasattr(req, 'bytes_transferred')) self.assertEquals(req.bytes_transferred, 10) @@ -1464,7 +1464,7 @@ class TestObjectController(unittest.TestCase): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/a/c/o') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1479,7 +1479,7 @@ class TestObjectController(unittest.TestCase): req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'Content-Length': '10'}, body='12345') - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(req.bytes_transferred, 5) self.assert_(hasattr(req, 'client_disconnect')) @@ -1492,7 +1492,7 @@ class TestObjectController(unittest.TestCase): controller = proxy_server.ObjectController(self.app, 'account', 'container', 'object') req = Request.blank('/a/c/o') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 5 @@ -1528,14 +1528,14 @@ class TestContainerController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c/', headers={'Content-Length': '0', 'Content-Type': 'text/plain'}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -1547,7 +1547,7 @@ class TestContainerController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(*statuses, **kwargs) self.app.memcache.store = {} req = Request.blank('/a/c', {}) - req.account = 'a' + self.app.update_request(req) res = controller.HEAD(req) self.assertEquals(res.status[:len(str(expected))], str(expected)) @@ -1570,7 +1570,7 @@ class TestContainerController(unittest.TestCase): self.app.memcache.store = {} req = Request.blank('/a/c', {}) req.content_length = 0 - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) @@ -1613,7 +1613,7 @@ class TestContainerController(unittest.TestCase): fake_http_connect(200, 200, 200, 200) self.app.memcache.store = {} req = Request.blank('/a/c', environ={'REQUEST_METHOD': meth}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, meth)(req) self.assertEquals(resp.status_int, 200) @@ -1657,7 +1657,7 @@ class TestContainerController(unittest.TestCase): self.app.memcache = MockMemcache(allow_lock=True) proxy_server.http_connect = fake_http_connect(200, 200, 200, 201, 201, 201, missing_container=True) req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}) - req.account = 'a' + self.app.update_request(req) res = controller.PUT(req) self.assertEquals(res.status_int, 201) @@ -1703,7 +1703,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c?format=json') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1715,7 +1715,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c?format=json') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 1 @@ -1760,8 +1760,7 @@ class TestContainerController(unittest.TestCase): 201, give_connect=test_connect) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={test_header: test_value}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) res = getattr(controller, method)(req) self.assertEquals(test_errors, []) @@ -1776,7 +1775,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'a', 'c') proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}) - req.account = 'a' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) @@ -1784,16 +1783,14 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1801,16 +1798,14 @@ class TestContainerController(unittest.TestCase): req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers={'X-Container-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1820,8 +1815,7 @@ class TestContainerController(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -1830,8 +1824,7 @@ class TestContainerController(unittest.TestCase): headers['X-Container-Meta-%d' % x] = 'v' req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1849,8 +1842,7 @@ class TestContainerController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 201) proxy_server.http_connect = fake_http_connect(201, 201, 201) @@ -1858,8 +1850,7 @@ class TestContainerController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}, headers=headers) - req.account = 'a' - req.container = 'c' + self.app.update_request(req) resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) @@ -1875,12 +1866,12 @@ class TestAccountController(unittest.TestCase): with save_globals(): proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a', {}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) proxy_server.http_connect = fake_http_connect(*statuses) req = Request.blank('/a/', {}) - req.account = 'a' + self.app.update_request(req) res = method(req) self.assertEquals(res.status_int, expected) @@ -1930,7 +1921,7 @@ class TestAccountController(unittest.TestCase): dev['port'] = 1 ## can't connect on this port controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) @@ -1941,7 +1932,7 @@ class TestAccountController(unittest.TestCase): dev['port'] = -1 ## invalid port number controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/account', environ={'REQUEST_METHOD': 'HEAD'}) - req.account = 'account' + self.app.update_request(req) resp = controller.HEAD(req) self.assertEquals(resp.status_int, 503) @@ -1950,7 +1941,7 @@ class TestAccountController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/a?format=json') - req.account = 'a' + self.app.update_request(req) res = controller.GET(req) res.body self.assert_(hasattr(res, 'bytes_transferred')) @@ -1961,7 +1952,7 @@ class TestAccountController(unittest.TestCase): proxy_server.http_connect = fake_http_connect(200, 200, body='{}') controller = proxy_server.AccountController(self.app, 'account') req = Request.blank('/a?format=json') - req.account = 'a' + self.app.update_request(req) orig_object_chunk_size = self.app.object_chunk_size try: self.app.object_chunk_size = 1 @@ -1999,7 +1990,7 @@ class TestAccountController(unittest.TestCase): give_connect=test_connect) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={test_header: test_value}) - req.account = 'a' + self.app.update_request(req) res = controller.POST(req) self.assertEquals(test_errors, []) @@ -2008,7 +1999,7 @@ class TestAccountController(unittest.TestCase): controller = proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) @@ -2016,14 +2007,14 @@ class TestAccountController(unittest.TestCase): req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-' + ('a' * MAX_META_NAME_LENGTH): 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-' + ('a' * (MAX_META_NAME_LENGTH + 1)): 'v'}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2031,14 +2022,14 @@ class TestAccountController(unittest.TestCase): req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-Too-Long': 'a' * MAX_META_VALUE_LENGTH}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Account-Meta-Too-Long': 'a' * (MAX_META_VALUE_LENGTH + 1)}) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2048,7 +2039,7 @@ class TestAccountController(unittest.TestCase): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) @@ -2057,7 +2048,7 @@ class TestAccountController(unittest.TestCase): headers['X-Account-Meta-%d' % x] = 'v' req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) @@ -2075,7 +2066,7 @@ class TestAccountController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size - 1) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 204) proxy_server.http_connect = fake_http_connect(204, 204, 204) @@ -2083,7 +2074,7 @@ class TestAccountController(unittest.TestCase): 'a' * (MAX_META_OVERALL_SIZE - size) req = Request.blank('/a', environ={'REQUEST_METHOD': 'POST'}, headers=headers) - req.account = 'a' + self.app.update_request(req) resp = controller.POST(req) self.assertEquals(resp.status_int, 400) From 5bc42442a978180c14218af9fec26494a6b84abb Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Fri, 3 Sep 2010 10:17:55 -0500 Subject: [PATCH 02/16] updated skipped tests in test.unit.obj.test_server to raise nose.SkipTest --- setup.cfg | 5 +++++ test/unit/obj/test_server.py | 37 +++++++++++++++++----------------- test/unit/proxy/test_server.py | 3 ++- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/setup.cfg b/setup.cfg index d53addcbf3..6671e0c70f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,3 +7,8 @@ source-dir = doc/source tag_build = tag_date = 0 tag_svn_revision = 0 + +[nosetests] +with-coverage=1 +cover-package=swift +verbosity=2 diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 17308c614a..b165c107d3 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -19,6 +19,7 @@ import cPickle as pickle import os import sys import unittest +from nose import SkipTest from shutil import rmtree from StringIO import StringIO from time import gmtime, sleep, strftime, time @@ -64,7 +65,7 @@ class TestObjectController(unittest.TestCase): def test_POST_update_meta(self): """ Test swift.object_server.ObjectController.POST """ if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -92,7 +93,7 @@ class TestObjectController(unittest.TestCase): def test_POST_not_exist(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/fail', environ={'REQUEST_METHOD': 'POST'}, headers={'X-Timestamp': timestamp, @@ -114,7 +115,7 @@ class TestObjectController(unittest.TestCase): def test_POST_container_connection(self): if not self.path_to_test_xfs: - return + raise SkipTest def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): @@ -210,7 +211,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_common(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -267,7 +268,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_no_etag(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Type': 'text/plain'}) @@ -286,7 +287,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_user_metadata(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': timestamp, @@ -314,7 +315,7 @@ class TestObjectController(unittest.TestCase): def test_PUT_container_connection(self): if not self.path_to_test_xfs: - return + raise SkipTest def mock_http_connect(response, with_exc=False): class FakeConn(object): def __init__(self, status, with_exc): @@ -443,7 +444,7 @@ class TestObjectController(unittest.TestCase): def test_GET(self): """ Test swift.object_server.ObjectController.GET """ if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c') resp = self.object_controller.GET(req) self.assertEquals(resp.status_int, 400) @@ -532,7 +533,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_match(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), @@ -586,7 +587,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_none_match(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ 'X-Timestamp': normalize_timestamp(time()), @@ -637,7 +638,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_modified_since(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ @@ -674,7 +675,7 @@ class TestObjectController(unittest.TestCase): def test_GET_if_unmodified_since(self): if not self.path_to_test_xfs: - return + raise SkipTest timestamp = normalize_timestamp(time()) req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={ @@ -713,7 +714,7 @@ class TestObjectController(unittest.TestCase): def test_DELETE(self): """ Test swift.object_server.ObjectController.DELETE """ if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'DELETE'}) resp = self.object_controller.DELETE(req) self.assertEquals(resp.status_int, 400) @@ -840,7 +841,7 @@ class TestObjectController(unittest.TestCase): def test_chunked_put(self): if not self.path_to_test_xfs: - return + raise SkipTest listener = listen(('localhost', 0)) port = listener.getsockname()[1] killer = spawn(wsgi.server, listener, self.object_controller, @@ -887,7 +888,7 @@ class TestObjectController(unittest.TestCase): def test_disk_file_app_iter_corners(self): if not self.path_to_test_xfs: - return + raise SkipTest df = object_server.DiskFile(self.testdir, 'sda1', '0', 'a', 'c', 'o') mkdirs(df.datadir) f = open(os.path.join(df.datadir, @@ -920,7 +921,7 @@ class TestObjectController(unittest.TestCase): def test_max_upload_time(self): if not self.path_to_test_xfs: - return + raise SkipTest class SlowBody(): def __init__(self): self.sent = 0 @@ -962,7 +963,7 @@ class TestObjectController(unittest.TestCase): def test_bad_sinces(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain'}, @@ -988,7 +989,7 @@ class TestObjectController(unittest.TestCase): def test_content_encoding(self): if not self.path_to_test_xfs: - return + raise SkipTest req = Request.blank('/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, headers={'X-Timestamp': normalize_timestamp(time()), 'Content-Length': '4', 'Content-Type': 'text/plain', diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index ab56dc740a..ce75c2014d 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -19,6 +19,7 @@ import logging import os import sys import unittest +from nose import SkipTest from ConfigParser import ConfigParser from contextlib import contextmanager from cStringIO import StringIO @@ -1052,7 +1053,7 @@ class TestObjectController(unittest.TestCase): 'pointing to a valid directory.\n' \ 'Please set PATH_TO_TEST_XFS to a directory on an XFS file ' \ 'system for testing.' - return + raise SkipTest testdir = \ os.path.join(path_to_test_xfs, 'tmp_test_proxy_server_chunked') mkdirs(testdir) From 92b4ceaed9618fec47baa802db29a095470fbc16 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 3 Sep 2010 17:54:30 -0700 Subject: [PATCH 03/16] Removed nosetests from setup.cfg as it causes coverage to be done with functtests and probetests, where it doesn't make much sense --- setup.cfg | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6671e0c70f..d53addcbf3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,8 +7,3 @@ source-dir = doc/source tag_build = tag_date = 0 tag_svn_revision = 0 - -[nosetests] -with-coverage=1 -cover-package=swift -verbosity=2 From 28ac96b90dead2b0806696e548c7068908d493f3 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 3 Sep 2010 20:47:43 -0700 Subject: [PATCH 04/16] Simply auth middleware and document how to make your own --- doc/source/development_auth.rst | 449 +++++++++++++++++++++++ doc/source/index.rst | 1 + swift/auth/server.py | 4 +- swift/common/middleware/acl.py | 76 ++++ swift/common/middleware/auth.py | 291 +++------------ test/unit/common/middleware/test_acl.py | 150 ++++++++ test/unit/common/middleware/test_auth.py | 52 +-- 7 files changed, 731 insertions(+), 292 deletions(-) create mode 100644 doc/source/development_auth.rst create mode 100644 swift/common/middleware/acl.py create mode 100644 test/unit/common/middleware/test_acl.py diff --git a/doc/source/development_auth.rst b/doc/source/development_auth.rst new file mode 100644 index 0000000000..fce52187b5 --- /dev/null +++ b/doc/source/development_auth.rst @@ -0,0 +1,449 @@ +=============== +Auth Middleware +=============== + +--------------------------------- +Creating Your Own Auth Middleware +--------------------------------- + +The included swift/common/middleware/auth.py is a good minimal example of how +to create auth middleware. The main points are that the auth middleware can +reject requests up front, before they ever get to the Swift Proxy application, +and afterwards when the proxy issues callbacks to verify authorization. + +It's generally good to separate the authentication and authorization +procedures. Authentication verifies that a request actually comes from who it +says it does. Authorization verifies the 'who' has access to the resource(s) +the request wants. + +Authentication is performed on the request before it ever gets to the Swift +Proxy application. The identity information is gleaned from the request, +validated in some way, and the validation information is added to the WSGI +environment as needed by the future authorization procedure. What exactly is +added to the WSGI environment is solely dependent on what the installed +authorization procedures need; the Swift Proxy application itself needs no +specific information, it just passes it along. Convention has +environ['REMOTE_USER'] set to the authenticated user string but often more +information is needed than just that. + +Authorization is performed through callbacks by the Swift Proxy server to the +WSGI environment's swift.authorize value, if one is set. The swift.authorize +value should simply be a function that takes a webob.Request as an argument and +returns None if access is granted or returns a callable(environ, +start_response) if access is denied. This callable is a standard WSGI callable. +Generally, you should return 403 Forbidden for requests by an authenticated +user and 401 Unauthorized for an unauthenticated request. For example, here's +an authorize function that only allows GETs (in this case you'd probably return +405 Method Not Allowed, but ignore that for the moment).:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + def authorize(req): + if req.method == 'GET': + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + +Adding the swift.authorize callback is often done by the authentication +middleware as authentication and authorization are often paired together. But, +you could create separate authorization middleware that simply sets the +callback before passing on the request. To continue our example above:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + if req.method == 'GET': + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The Swift Proxy server will call swift.authorize after some initial work, but +before truly trying to process the request. Positive authorization at this +point will cause the request to be fully processed immediately. A denial at +this point will immediately send the denial response for most operations. + +But for some operations that might be approved with more information, the +additional information will be gathered and added to the WSGI environment and +then swift.authorize will be called once more. These are called delay_denial +requests and currently include container read requests and object read and +write requests. For these requests, the read or write access control string +(X-Container-Read and X-Container-Write) will be fetched and set as the 'acl' +attribute in the webob.Request passed to swift.authorize. + +The delay_denial procedures allow skipping possibly expensive access control +string retrievals for requests that can be approved without that information, +such as administrator or account owner requests. + +To further our example, we now will approve all requests that have the access +control string set to same value as the authenticated user string. Note that +you probably wouldn't do this exactly as the access control string represents a +list rather than a single user, but it'll suffice for this example:: + + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + # Allow anyone to perform GET requests + if req.method == 'GET': + return None + # Allow any request where the acl equals the authenticated user + if getattr(req, 'acl', None) == req.remote_user: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The access control string has a standard format included with Swift, though +this can be overridden if desired. The standard format can be parsed with +swift.common.middleware.acl.parse_acl which converts the string into two arrays +of strings: (referrers, groups). The referrers allow comparing the request's +Referer header to control access. The groups allow comparing the +request.remote_user (or other sources of group information) to control access. +Checking referrer access can be accomplished by using the +swift.common.middleware.acl.referrer_allowed function. Checking group access is +usually a simple string comparison. + +Let's continue our example to use parse_acl and referrer_allowed. Now we'll +only allow GETs after a referrer check and any requests after a group check:: + + from swift.common.middleware.acl import parse_acl, referrer_allowed + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + return self.app(environ, start_response) + + def authorize(self, req): + if hasattr(req, 'acl'): + referrers, groups = parse_acl(req.acl) + if req.method == 'GET' and referrer_allowed(req, referrers): + return None + if req.remote_user and groups and req.remote_user in groups: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +The access control strings are set with PUTs and POSTs to containers with the +X-Container-Read and X-Container-Write headers. Swift allows these strings to +be set to any value, though it's very useful to validate the strings meet the +desired format and return a useful error to the user if they don't. + +To support this validation, the Swift Proxy application will call the WSGI +environment's swift.clean_acl callback whenever one of these headers is to be +written. The callback should take a header name and value as its arguments. It +should return the cleaned value to save if valid or raise a ValueError with a +reasonable error message if not. + +There is an included swift.common.middleware.acl.clean_acl that validates the +standard Swift format. Let's improve our example by making use of that:: + + from swift.common.middleware.acl import \ + clean_acl, parse_acl, referrer_allowed + from webob import HTTPForbidden, HTTPUnauthorized + + + class Authorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = clean_acl + return self.app(environ, start_response) + + def authorize(self, req): + if hasattr(req, 'acl'): + referrers, groups = parse_acl(req.acl) + if req.method == 'GET' and referrer_allowed(req, referrers): + return None + if req.remote_user and groups and req.remote_user in groups: + return None + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return Authorization(app, conf) + return auth_filter + +Now, if you want to override the format for access control strings you'll have +to provide your own clean_acl function and you'll have to do your own parsing +and authorization checking for that format. It's highly recommended you use the +standard format simply to support the widest range of external tools, but +sometimes that's less important than meeting certain ACL requirements. + + +---------------------------- +Integrating With repoze.what +---------------------------- + +Here's an example of integration with repoze.what, though honestly it just does +what the default swift/common/middleware/auth.py does in a slightly different +way. I'm no repoze.what expert by any stretch; this is just included here to +hopefully give folks a start on their own code if they want to use +repoze.what:: + + from time import time + + from eventlet.timeout import Timeout + from repoze.what.adapters import BaseSourceAdapter + from repoze.what.middleware import setup_auth + from repoze.what.predicates import in_any_group, NotAuthorizedError + from swift.common.bufferedhttp import http_connect_raw as http_connect + from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed + from swift.common.utils import cache_from_env, split_path + from webob.exc import HTTPForbidden, HTTPUnauthorized + + + class DevAuthorization(object): + + def __init__(self, app, conf): + self.app = app + self.conf = conf + + def __call__(self, environ, start_response): + environ['swift.authorize'] = self.authorize + environ['swift.clean_acl'] = clean_acl + return self.app(environ, start_response) + + def authorize(self, req): + version, account, container, obj = split_path(req.path, 1, 4, True) + if not account: + return self.denied_response(req) + referrers, groups = parse_acl(getattr(req, 'acl', None)) + if referrer_allowed(req, referrers): + return None + try: + in_any_group(account, *groups).check_authorization(req.environ) + except NotAuthorizedError: + return self.denied_response(req) + return None + + def denied_response(self, req): + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) + + + class DevIdentifier(object): + + def __init__(self, conf): + self.conf = conf + + def identify(self, env): + return {'token': + env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))} + + def remember(self, env, identity): + return [] + + def forget(self, env, identity): + return [] + + + class DevAuthenticator(object): + + def __init__(self, conf): + self.conf = conf + self.auth_host = conf.get('ip', '127.0.0.1') + self.auth_port = int(conf.get('port', 11000)) + self.ssl = \ + conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes') + self.timeout = int(conf.get('node_timeout', 10)) + + def authenticate(self, env, identity): + token = identity.get('token') + if not token: + return None + memcache_client = cache_from_env(env) + key = 'devauth/%s' % token + cached_auth_data = memcache_client.get(key) + if cached_auth_data: + start, expiration, user = cached_auth_data + if time() - start <= expiration: + return user + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s' % token, ssl=self.ssl) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status == 204: + expiration = float(resp.getheader('x-auth-ttl')) + user = resp.getheader('x-auth-user') + memcache_client.set(key, (time(), expiration, user), + timeout=expiration) + return user + return None + + + class DevChallenger(object): + + def __init__(self, conf): + self.conf = conf + + def challenge(self, env, status, app_headers, forget_headers): + def no_challenge(env, start_response): + start_response(str(status), []) + return [] + return no_challenge + + + class DevGroupSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevGroupSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, credentials): + return credentials['repoze.what.userid'].split(',') + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + + + class DevPermissionSourceAdapter(BaseSourceAdapter): + + def __init__(self, *args, **kwargs): + super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs) + self.sections = {} + + def _get_all_sections(self): + return self.sections + + def _get_section_items(self, section): + return self.sections[section] + + def _find_sections(self, group_name): + return set([n for (n, p) in self.sections.items() + if group_name in p]) + + def _include_items(self, section, items): + self.sections[section] |= items + + def _exclude_items(self, section, items): + for item in items: + self.sections[section].remove(item) + + def _item_is_included(self, section, item): + return item in self.sections[section] + + def _create_section(self, section): + self.sections[section] = set() + + def _edit_section(self, section, new_section): + self.sections[new_section] = self.sections[section] + del self.sections[section] + + def _delete_section(self, section): + del self.sections[section] + + def _section_exists(self, section): + return self.sections.has_key(section) + + + def filter_factory(global_conf, **local_conf): + conf = global_conf.copy() + conf.update(local_conf) + def auth_filter(app): + return setup_auth(DevAuthorization(app, conf), + group_adapters={'all_groups': DevGroupSourceAdapter()}, + permission_adapters={'all_perms': DevPermissionSourceAdapter()}, + identifiers=[('devauth', DevIdentifier(conf))], + authenticators=[('devauth', DevAuthenticator(conf))], + challengers=[('devauth', DevChallenger(conf))]) + return auth_filter diff --git a/doc/source/index.rst b/doc/source/index.rst index 7e4681ca28..8760852f13 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,6 +32,7 @@ Development: development_guidelines development_saio + development_auth Deployment: diff --git a/swift/auth/server.py b/swift/auth/server.py index db290a72c4..39c8dd8837 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -353,8 +353,10 @@ class AuthController(object): validation = self.validate_token(token) if not validation: return HTTPNotFound() + # X-Auth-User: account:user,account,cfaccount return HTTPNoContent(headers={'X-Auth-TTL': validation[0], - 'X-Auth-User': ':'.join(validation[1:])}) + 'X-Auth-User': '%s:%s,%s,%s' % + (validation[1], validation[2], validation[1], validation[3])}) def handle_account_create(self, request): """ diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py new file mode 100644 index 0000000000..bfa88b8105 --- /dev/null +++ b/swift/common/middleware/acl.py @@ -0,0 +1,76 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def clean_acl(name, value): + values = [] + for raw_value in value.lower().split(','): + raw_value = raw_value.strip() + if raw_value: + if ':' in raw_value: + first, second = (v.strip() for v in raw_value.split(':', 1)) + if not first: + raise ValueError('No value before colon in %s' % + repr(raw_value)) + if first == '.ref' and 'write' in name: + raise ValueError('Referrers not allowed in write ACLs: %s' + % repr(raw_value)) + if second: + if first == '.ref' and second[0] == '-': + second = second[1:].strip() + if not second: + raise ValueError('No value after referrer deny ' + 'designation in %s' % repr(raw_value)) + second = '-' + second + values.append('%s:%s' % (first, second)) + elif first == '.ref': + raise ValueError('No value after referrer designation in ' + '%s' % repr(raw_value)) + else: + values.append(first) + else: + values.append(raw_value) + return ','.join(values) + + +def parse_acl(acl_string): + referrers = [] + groups = [] + if acl_string: + for value in acl_string.split(','): + if value.startswith('.ref:'): + referrers.append(value[len('.ref:'):]) + else: + groups.append(value) + return referrers, groups + + +def referrer_allowed(req, referrers): + allow = False + if referrers: + parts = req.referer.split('//', 1) + if len(parts) == 2: + rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() + else: + rhost = 'unknown' + for mhost in referrers: + if mhost[0] == '-': + mhost = mhost[1:] + if mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = False + elif mhost == 'any' or mhost == rhost or \ + (mhost[0] == '.' and rhost.endswith(mhost)): + allow = True + return allow diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index 32800839ba..aee5e99353 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -16,268 +16,79 @@ from time import time from eventlet.timeout import Timeout -from repoze.what.adapters import BaseSourceAdapter -from repoze.what.middleware import setup_auth -from repoze.what.predicates import in_any_group, NotAuthorizedError from webob.exc import HTTPForbidden, HTTPUnauthorized from swift.common.bufferedhttp import http_connect_raw as http_connect +from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed from swift.common.utils import cache_from_env, split_path -class DevAuthorization(object): +class DevAuth(object): def __init__(self, app, conf): self.app = app self.conf = conf - - def __call__(self, environ, start_response): - environ['swift.authorize'] = self.authorize - environ['swift.clean_acl'] = self.clean_acl - return self.app(environ, start_response) - - def authorize(self, req): - version, account, container, obj = split_path(req.path, 1, 4, True) - if not account: - return self.denied_response(req) - groups = [account] - acl = self.parse_acl(getattr(req, 'acl', None)) - if acl: - referrers, accounts, users = acl - if referrers: - parts = req.referer.split('//', 1) - allow = False - if len(parts) == 2: - rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() - else: - rhost = 'unknown' - for mhost in referrers: - if mhost[0] == '-': - mhost = mhost[1:] - if mhost == rhost or \ - (mhost[0] == '.' and rhost.endswith(mhost)): - allow = False - elif mhost == 'any' or mhost == rhost or \ - (mhost[0] == '.' and rhost.endswith(mhost)): - allow = True - if allow: - return None - groups.extend(accounts) - groups.extend(users) - try: - in_any_group(*groups).check_authorization(req.environ) - except NotAuthorizedError: - return self.denied_response(req) - return None - - def denied_response(self, req): - if req.remote_user: - return HTTPForbidden(request=req) - else: - return HTTPUnauthorized(request=req) - - def clean_acl(self, header_name, value): - values = [] - for raw_value in value.lower().split(','): - raw_value = raw_value.strip() - if raw_value: - if ':' in raw_value: - first, second = \ - (v.strip() for v in raw_value.split(':', 1)) - if not first: - raise ValueError('No value before colon in %s' % - repr(raw_value)) - if first == '.ref' and 'write' in header_name: - raise ValueError('Referrers not allowed in write ' - 'ACLs: %s' % repr(raw_value)) - if second: - if first == '.ref' and second[0] == '-': - second = second[1:].strip() - if not second: - raise ValueError('No value after referrer ' - 'deny designation in %s' % repr(raw_value)) - second = '-' + second - values.append('%s:%s' % (first, second)) - elif first == '.ref': - raise ValueError('No value after referrer designation ' - 'in %s' % repr(raw_value)) - else: - values.append(first) - else: - values.append(raw_value) - return ','.join(values) - - def parse_acl(self, acl_string): - if not acl_string: - return None - referrers = [] - accounts = [] - users = [] - for value in acl_string.split(','): - if value.startswith('.ref:'): - referrers.append(value[len('.ref:'):]) - elif ':' in value: - users.append(value) - else: - accounts.append(value) - return (referrers, accounts, users) - - -class DevIdentifier(object): - - def __init__(self, conf): - self.conf = conf - - def identify(self, env): - return {'token': - env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))} - - def remember(self, env, identity): - return [] - - def forget(self, env, identity): - return [] - - -class DevAuthenticator(object): - - def __init__(self, conf): - self.conf = conf self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) self.ssl = \ conf.get('ssl', 'false').lower() in ('true', 'on', '1', 'yes') self.timeout = int(conf.get('node_timeout', 10)) - def authenticate(self, env, identity): - token = identity.get('token') - if not token: + def __call__(self, env, start_response): + user = None + token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) + if token: + memcache_client = cache_from_env(env) + key = 'devauth/%s' % token + cached_auth_data = memcache_client.get(key) + if cached_auth_data: + start, expiration, user = cached_auth_data + if time() - start > expiration: + user = None + if not user: + with Timeout(self.timeout): + conn = http_connect(self.auth_host, self.auth_port, 'GET', + '/token/%s' % token, ssl=self.ssl) + resp = conn.getresponse() + resp.read() + conn.close() + if resp.status // 100 != 2: + return HTTPUnauthorized()(env, start_response) + expiration = float(resp.getheader('x-auth-ttl')) + user = resp.getheader('x-auth-user') + memcache_client.set(key, (time(), expiration, user), + timeout=expiration) + env['REMOTE_USER'] = user + env['swift.authorize'] = self.authorize + env['swift.clean_acl'] = clean_acl + return self.app(env, start_response) + + def authorize(self, req): + version, account, container, obj = split_path(req.path, 1, 4, True) + if not account: + return self.denied_response(req) + if req.remote_user and account in req.remote_user.split(','): return None - memcache_client = cache_from_env(env) - key = 'devauth/%s' % token - cached_auth_data = memcache_client.get(key) - if cached_auth_data: - start, expiration, user = cached_auth_data - if time() - start <= expiration: - return user - with Timeout(self.timeout): - conn = http_connect(self.auth_host, self.auth_port, 'GET', - '/token/%s' % token, ssl=self.ssl) - resp = conn.getresponse() - resp.read() - conn.close() - if resp.status == 204: - expiration = float(resp.getheader('x-auth-ttl')) - user = resp.getheader('x-auth-user') - memcache_client.set(key, (time(), expiration, user), - timeout=expiration) - return user - return None + referrers, groups = parse_acl(getattr(req, 'acl', None)) + if referrer_allowed(req, referrers): + return None + if not req.remote_user: + return self.denied_response(req) + for user_group in req.remote_user.split(','): + if user_group in groups: + return None + return self.denied_response(req) - -class DevChallenger(object): - - def __init__(self, conf): - self.conf = conf - - def challenge(self, env, status, app_headers, forget_headers): - def no_challenge(env, start_response): - start_response(str(status), []) - return [] - return no_challenge - - -class DevGroupSourceAdapter(BaseSourceAdapter): - - def __init__(self, *args, **kwargs): - super(DevGroupSourceAdapter, self).__init__(*args, **kwargs) - self.sections = {} - - def _get_all_sections(self): - return self.sections - - def _get_section_items(self, section): - return self.sections[section] - - def _find_sections(self, credentials): - creds = credentials['repoze.what.userid'].split(':') - if len(creds) != 3: - return set() - rv = set([creds[0], ':'.join(creds[:2]), creds[2]]) - return rv - - def _include_items(self, section, items): - self.sections[section] |= items - - def _exclude_items(self, section, items): - for item in items: - self.sections[section].remove(item) - - def _item_is_included(self, section, item): - return item in self.sections[section] - - def _create_section(self, section): - self.sections[section] = set() - - def _edit_section(self, section, new_section): - self.sections[new_section] = self.sections[section] - del self.sections[section] - - def _delete_section(self, section): - del self.sections[section] - - def _section_exists(self, section): - return self.sections.has_key(section) - - -class DevPermissionSourceAdapter(BaseSourceAdapter): - - def __init__(self, *args, **kwargs): - super(DevPermissionSourceAdapter, self).__init__(*args, **kwargs) - self.sections = {} - - def _get_all_sections(self): - return self.sections - - def _get_section_items(self, section): - return self.sections[section] - - def _find_sections(self, group_name): - return set([n for (n, p) in self.sections.items() - if group_name in p]) - - def _include_items(self, section, items): - self.sections[section] |= items - - def _exclude_items(self, section, items): - for item in items: - self.sections[section].remove(item) - - def _item_is_included(self, section, item): - return item in self.sections[section] - - def _create_section(self, section): - self.sections[section] = set() - - def _edit_section(self, section, new_section): - self.sections[new_section] = self.sections[section] - del self.sections[section] - - def _delete_section(self, section): - del self.sections[section] - - def _section_exists(self, section): - return self.sections.has_key(section) + def denied_response(self, req): + if req.remote_user: + return HTTPForbidden(request=req) + else: + return HTTPUnauthorized(request=req) def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) def auth_filter(app): - return setup_auth(DevAuthorization(app, conf), - group_adapters={'all_groups': DevGroupSourceAdapter()}, - permission_adapters={'all_perms': DevPermissionSourceAdapter()}, - identifiers=[('devauth', DevIdentifier(conf))], - authenticators=[('devauth', DevAuthenticator(conf))], - challengers=[('devauth', DevChallenger(conf))]) + return DevAuth(app, conf) return auth_filter diff --git a/test/unit/common/middleware/test_acl.py b/test/unit/common/middleware/test_acl.py new file mode 100644 index 0000000000..69de606849 --- /dev/null +++ b/test/unit/common/middleware/test_acl.py @@ -0,0 +1,150 @@ +# Copyright (c) 2010 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +import logging +import os +import sys +import unittest +from contextlib import contextmanager + +import eventlet +from webob import Request + +from swift.common.middleware import acl + +# mocks +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + + +class FakeMemcache(object): + def __init__(self): + self.store = {} + + def get(self, key): + return self.store.get(key) + + def set(self, key, value, timeout=0): + self.store[key] = value + return True + + def incr(self, key, timeout=0): + self.store[key] = self.store.setdefault(key, 0) + 1 + return self.store[key] + + @contextmanager + def soft_lock(self, key, timeout=0, retries=5): + yield True + + def delete(self, key): + try: + del self.store[key] + except: + pass + return True + + +def mock_http_connect(response, headers=None, with_exc=False): + class FakeConn(object): + def __init__(self, status, headers, with_exc): + self.status = status + self.reason = 'Fake' + self.host = '1.2.3.4' + self.port = '1234' + self.with_exc = with_exc + self.headers = headers + if self.headers is None: + self.headers = {} + def getresponse(self): + if self.with_exc: + raise Exception('test') + return self + def getheader(self, header): + return self.headers[header] + def read(self, amt=None): + return '' + def close(self): + return + return lambda *args, **kwargs: FakeConn(response, headers, with_exc) + + +class Logger(object): + def __init__(self): + self.error_value = None + self.exception_value = None + def error(self, msg, *args, **kwargs): + self.error_value = (msg, args, kwargs) + def exception(self, msg, *args, **kwargs): + _, exc, _ = sys.exc_info() + self.exception_value = (msg, + '%s %s' % (exc.__class__.__name__, str(exc)), args, kwargs) +# tests + +class FakeApp(object): + def __call__(self, env, start_response): + return "OK" + +def start_response(*args): + pass + +class TestAuth(unittest.TestCase): + # I brought these over from another refactor I've been trying, but they + # need work. + + def test_clean_acl(self): + value = acl.clean_acl('header', '.ref:any') + self.assertEquals(value, '.ref:any') + value = acl.clean_acl('header', '.ref:specific.host') + self.assertEquals(value, '.ref:specific.host') + value = acl.clean_acl('header', '.ref:.ending.with') + self.assertEquals(value, '.ref:.ending.with') + value = acl.clean_acl('header', '.ref:one,.ref:two') + self.assertEquals(value, '.ref:one,.ref:two') + value = acl.clean_acl('header', '.ref:any,.ref:-specific.host') + self.assertEquals(value, '.ref:any,.ref:-specific.host') + value = acl.clean_acl('header', '.ref:any,.ref:-.ending.with') + self.assertEquals(value, '.ref:any,.ref:-.ending.with') + value = acl.clean_acl('header', '.ref:one,.ref:-two') + self.assertEquals(value, '.ref:one,.ref:-two') + value = acl.clean_acl('header', + ' .ref : one , ,, .ref:two , .ref : - three ') + self.assertEquals(value, '.ref:one,.ref:two,.ref:-three') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.ref:') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .ref : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', + 'user , .ref : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.ref:-') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .ref : - ') + self.assertRaises(ValueError, acl.clean_acl, 'header', + 'user , .ref : - ') + + def test_parse_acl(self): + self.assertEquals(acl.parse_acl(None), ([], [])) + self.assertEquals(acl.parse_acl(''), ([], [])) + self.assertEquals(acl.parse_acl('.ref:ref1'), (['ref1'], [])) + self.assertEquals(acl.parse_acl('.ref:-ref1'), (['-ref1'], [])) + self.assertEquals(acl.parse_acl('account:user'), + ([], ['account:user'])) + self.assertEquals(acl.parse_acl('account'), ([], ['account'])) + self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.ref:ref3,.ref:-ref4'), + (['ref3', '-ref4'], ['acc1', 'acc2:usr2'])) + self.assertEquals(acl.parse_acl( + 'acc1,acc2:usr2,.ref:ref3,acc3,acc4:usr4,.ref:ref5,.ref:-ref6'), + (['ref3', 'ref5', '-ref6'], + ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index 08f8898d1b..d48ce39202 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -101,57 +101,7 @@ def start_response(*args): class TestAuth(unittest.TestCase): # TODO: With the auth refactor, these tests have to be refactored as well. - # I brought some over from another refactor I've been trying, but these - # also need work. - - def test_clean_acl(self): - devauth = auth.DevAuthorization(None, None) - value = devauth.clean_acl('header', '.ref:any') - self.assertEquals(value, '.ref:any') - value = devauth.clean_acl('header', '.ref:specific.host') - self.assertEquals(value, '.ref:specific.host') - value = devauth.clean_acl('header', '.ref:.ending.with') - self.assertEquals(value, '.ref:.ending.with') - value = devauth.clean_acl('header', '.ref:one,.ref:two') - self.assertEquals(value, '.ref:one,.ref:two') - value = devauth.clean_acl('header', '.ref:any,.ref:-specific.host') - self.assertEquals(value, '.ref:any,.ref:-specific.host') - value = devauth.clean_acl('header', '.ref:any,.ref:-.ending.with') - self.assertEquals(value, '.ref:any,.ref:-.ending.with') - value = devauth.clean_acl('header', '.ref:one,.ref:-two') - self.assertEquals(value, '.ref:one,.ref:-two') - value = devauth.clean_acl('header', - ' .ref : one , ,, .ref:two , .ref : - three ') - self.assertEquals(value, '.ref:one,.ref:two,.ref:-three') - self.assertRaises(ValueError, devauth.clean_acl, 'header', '.ref:') - self.assertRaises(ValueError, devauth.clean_acl, 'header', ' .ref : ') - self.assertRaises(ValueError, devauth.clean_acl, 'header', - 'user , .ref : ') - self.assertRaises(ValueError, devauth.clean_acl, 'header', '.ref:-') - self.assertRaises(ValueError, devauth.clean_acl, 'header', - ' .ref : - ') - self.assertRaises(ValueError, devauth.clean_acl, 'header', - 'user , .ref : - ') - - def test_parse_acl(self): - devauth = auth.DevAuthorization(None, None) - self.assertEquals(devauth.parse_acl(None), None) - self.assertEquals(devauth.parse_acl(''), None) - self.assertEquals(devauth.parse_acl('.ref:ref1'), - (['ref1'], [], [])) - self.assertEquals(devauth.parse_acl('.ref:-ref1'), - (['-ref1'], [], [])) - self.assertEquals(devauth.parse_acl('account:user'), - ([], [], ['account:user'])) - self.assertEquals(devauth.parse_acl('account'), - ([], ['account'], [])) - self.assertEquals( - devauth.parse_acl('acc1,acc2:usr2,.ref:ref3,.ref:-ref4'), - (['ref3', '-ref4'], ['acc1'], ['acc2:usr2'])) - self.assertEquals(devauth.parse_acl( - 'acc1,acc2:usr2,.ref:ref3,acc3,acc4:usr4,.ref:ref5,.ref:-ref6'), - (['ref3', 'ref5', '-ref6'], ['acc1', 'acc3'], - ['acc2:usr2', 'acc4:usr4'])) + pass if __name__ == '__main__': From bb01c2244051f653c0f226ae01dd0371d369c1ee Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 3 Sep 2010 21:39:44 -0700 Subject: [PATCH 05/16] Updated tools and client.py to work with ACLs --- bin/st | 450 +++++++++++++----- bin/swift-stats-populate | 3 +- bin/swift-stats-report | 39 +- swift/common/client.py | 203 +++++--- test/probe/test_account_failures.py | 46 +- test/probe/test_container_failures.py | 104 ++-- test/probe/test_object_handoff.py | 11 +- .../probe/test_running_with_each_type_down.py | 27 +- test/unit/common/test_client.py | 9 +- test/unit/proxy/test_server.py | 4 +- 10 files changed, 609 insertions(+), 287 deletions(-) diff --git a/bin/st b/bin/st index ede5c985de..f7546b7472 100755 --- a/bin/st +++ b/bin/st @@ -154,19 +154,26 @@ except: def get_auth(url, user, key, snet=False): """ - Get authentication credentials + Get authentication/authorization credentials. - :param url: authentication URL - :param user: user to auth as - :param key: key or passowrd for auth - :param snet: use SERVICENET internal network default is False - :returns: tuple of (storage URL, storage token, auth token) + The snet parameter is used for Rackspace's ServiceNet internal network + implementation. In this function, it simply adds *snet-* to the beginning + of the host name for the returned storage URL. With Rackspace Cloud Files, + use of this network path causes no bandwidth charges but requires the + client to be running on Rackspace's ServiceNet network. + + :param url: authentication/authorization URL + :param user: user to authenticate as + :param key: key or password for authorization + :param snet: use SERVICENET internal network (see above), default is False + :returns: tuple of (storage URL, auth token) :raises ClientException: HTTP GET request to auth URL failed """ parsed, conn = http_connection(url) conn.request('GET', parsed.path, '', {'X-Auth-User': user, 'X-Auth-Key': key}) resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException('Auth GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, @@ -175,7 +182,7 @@ except: url = resp.getheader('x-storage-url') if snet: parsed = list(urlparse(url)) - # Second item in the list is the netloc + # Second item in the list is the netloc parsed[1] = 'snet-' + parsed[1] url = urlunparse(parsed) return url, resp.getheader('x-storage-token', @@ -196,18 +203,21 @@ except: conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of accounts + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_account(url, token, marker, limit, prefix, http_conn) + rv = get_account(url, token, marker, limit, prefix, http_conn) + listing = rv[1] while listing: - rv.extend(listing) marker = listing[-1]['name'] - listing = get_account(url, token, marker, limit, prefix, http_conn) + listing = \ + get_account(url, token, marker, limit, prefix, http_conn)[1] + if listing: + rv.extend(listing) return rv parsed, conn = http_conn qs = 'format=json' @@ -220,6 +230,9 @@ except: conn.request('GET', '%s?%s' % (parsed.path, qs), '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException('Account GET failed', http_scheme=parsed.scheme, @@ -228,8 +241,8 @@ except: http_reason=resp.reason) if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_account(url, token, http_conn=None): @@ -240,7 +253,8 @@ except: :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (container count, object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -249,14 +263,42 @@ except: parsed, conn = http_connection(url) conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-account-container-count', 0)), \ - int(resp.getheader('x-account-object-count', 0)), \ - int(resp.getheader('x-account-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + + def post_account(url, token, headers, http_conn=None): + """ + Update an account's metadata. + + :param url: storage URL + :param token: auth token + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + headers['X-Auth-Token'] = token + conn.request('POST', parsed.path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) def get_container(url, token, container, marker=None, limit=None, @@ -276,23 +318,25 @@ except: conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) + rv = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + listing = rv[1] while listing: - rv.extend(listing) if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn) + prefix, delimiter, http_conn)[1] + if listing: + rv[1].extend(listing) return rv parsed, conn = http_conn path = '%s/%s' % (parsed.path, quote(container)) @@ -313,10 +357,13 @@ except: http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_query=qs, http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_container(url, token, container, http_conn=None): @@ -328,7 +375,8 @@ except: :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -344,17 +392,20 @@ except: http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count', 0)), \ - int(resp.getheader('x-container-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers - def put_container(url, token, container, http_conn=None): + def put_container(url, token, container, headers=None, http_conn=None): """ Create a container :param url: storage URL :param token: auth token :param container: container name to create + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP PUT request failed @@ -364,7 +415,10 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) - conn.request('PUT', path, '', {'X-Auth-Token': token}) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + conn.request('PUT', path, '', headers) resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: @@ -374,6 +428,34 @@ except: http_reason=resp.reason) + def post_container(url, token, container, headers, http_conn=None): + """ + Update a container's metadata. + + :param url: storage URL + :param token: auth token + :param container: container name to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + def delete_container(url, token, container, http_conn=None): """ Delete a container @@ -411,8 +493,12 @@ except: :param name: object name to get :param http_conn: HTTP connection object (If None, it will create the conn object) - :param resp_chunk_size: if defined, chunk size of data to read - :returns: a list of objects + :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If + you specify a resp_chunk_size you must fully read + the object's contents before making another + request. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if http_conn: @@ -441,12 +527,10 @@ except: object_body = _object_body() else: object_body = resp.read() - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata, \ - object_body + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def head_object(url, token, container, name, http_conn=None): @@ -459,8 +543,8 @@ except: :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (content type, content length, last modfied, etag, - dictionary of metadata) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -479,16 +563,15 @@ except: for key, value in resp.getheaders(): if key.lower().startswith('x-object-meta-'): metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers - def put_object(url, token, container, name, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None, http_conn=None): + def put_object(url, token, container, name, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, headers=None, + http_conn=None): """ Put an object @@ -496,12 +579,12 @@ except: :param token: auth token :param container: container name that the object is in :param name: object name to put - :param contents: file like object to read object data from - :param metadata: dictionary of object metadata + :param contents: a string or a file like object to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param chunk_size: chunk size of data to write :param content_type: value to send as content-type header + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :returns: etag from server response @@ -512,9 +595,9 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if not headers: + headers = {} + headers['X-Auth-Token'] = token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -550,7 +633,7 @@ except: return resp.getheader('etag').strip('"') - def post_object(url, token, container, name, metadata, http_conn=None): + def post_object(url, token, container, name, headers, http_conn=None): """ Change object metadata @@ -558,7 +641,7 @@ except: :param token: auth token :param container: container name that the object is in :param name: object name to change - :param metadata: dictionary of object metadata + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP POST request failed @@ -568,9 +651,7 @@ except: else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + headers['X-Auth-Token'] = token conn.request('POST', path, '', headers) resp = conn.getresponse() resp.read() @@ -620,7 +701,7 @@ except: :param preauthurl: storage URL (if you have already authenticated) :param preauthtoken: authentication token (if you have already authenticated) - :param snet: use SERVICENET internal network default is False + :param snet: use SERVICENET internal network default is False """ self.authurl = authurl self.user = user @@ -632,20 +713,24 @@ except: self.attempts = 0 self.snet = snet + def get_auth(self): + return get_auth(self.authurl, self.user, self.key, snet=self.snet) + + def http_connection(self): + return http_connection(self.url) + def _retry(self, func, *args, **kwargs): - kwargs['http_conn'] = self.http_conn self.attempts = 0 backoff = 1 while self.attempts <= self.retries: self.attempts += 1 try: if not self.url or not self.token: - self.url, self.token = \ - get_auth(self.authurl, self.user, self.key, snet=self.snet) + self.url, self.token = self.get_auth() self.http_conn = None if not self.http_conn: - self.http_conn = http_connection(self.url) - kwargs['http_conn'] = self.http_conn + self.http_conn = self.http_connection() + kwargs['http_conn'] = self.http_conn rv = func(self.url, self.token, *args, **kwargs) return rv except (socket.error, HTTPException): @@ -667,63 +752,71 @@ except: backoff *= 2 def head_account(self): - """Wrapper for head_account""" + """Wrapper for :func:`head_account`""" return self._retry(head_account) def get_account(self, marker=None, limit=None, prefix=None, full_listing=False): - """Wrapper for get_account""" - # TODO: With full_listing=True this will restart the entire listing - # with each retry. Need to make a better version that just retries - # where it left off. + """Wrapper for :func:`get_account`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. return self._retry(get_account, marker=marker, limit=limit, prefix=prefix, full_listing=full_listing) + def post_account(self, headers): + """Wrapper for :func:`post_account`""" + return self._retry(post_account, headers) + def head_container(self, container): - """Wrapper for head_container""" + """Wrapper for :func:`head_container`""" return self._retry(head_container, container) def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, full_listing=False): - """Wrapper for get_container""" - # TODO: With full_listing=True this will restart the entire listing - # with each retry. Need to make a better version that just retries - # where it left off. + """Wrapper for :func:`get_container`""" + # TODO(unknown): With full_listing=True this will restart the entire + # listing with each retry. Need to make a better version that just + # retries where it left off. return self._retry(get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, full_listing=full_listing) - def put_container(self, container): - """Wrapper for put_container""" - return self._retry(put_container, container) + def put_container(self, container, headers=None): + """Wrapper for :func:`put_container`""" + return self._retry(put_container, container, headers=headers) + + def post_container(self, container, headers): + """Wrapper for :func:`post_container`""" + return self._retry(post_container, container, headers) def delete_container(self, container): - """Wrapper for delete_container""" + """Wrapper for :func:`delete_container`""" return self._retry(delete_container, container) def head_object(self, container, obj): - """Wrapper for head_object""" + """Wrapper for :func:`head_object`""" return self._retry(head_object, container, obj) def get_object(self, container, obj, resp_chunk_size=None): - """Wrapper for get_object""" + """Wrapper for :func:`get_object`""" return self._retry(get_object, container, obj, resp_chunk_size=resp_chunk_size) - def put_object(self, container, obj, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None): - """Wrapper for put_object""" + def put_object(self, container, obj, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, + headers=None): + """Wrapper for :func:`put_object`""" return self._retry(put_object, container, obj, contents, - metadata=metadata, content_length=content_length, etag=etag, - chunk_size=chunk_size, content_type=content_type) + content_length=content_length, etag=etag, chunk_size=chunk_size, + content_type=content_type, headers=headers) - def post_object(self, container, obj, metadata): - """Wrapper for post_object""" - return self._retry(post_object, container, obj, metadata) + def post_object(self, container, obj, headers): + """Wrapper for :func:`post_object`""" + return self._retry(post_object, container, obj, headers) def delete_object(self, container, obj): - """Wrapper for delete_object""" + """Wrapper for :func:`delete_object`""" return self._retry(delete_object, container, obj) # End inclusion of swift.common.client @@ -805,7 +898,7 @@ def st_delete(options, args): marker = '' while True: objects = [o['name'] for o in - conn.get_container(container, marker=marker)] + conn.get_container(container, marker=marker)[1]] if not objects: break for obj in objects: @@ -847,7 +940,7 @@ def st_delete(options, args): marker = '' while True: containers = \ - [c['name'] for c in conn.get_account(marker=marker)] + [c['name'] for c in conn.get_account(marker=marker)[1]] if not containers: break for container in containers: @@ -893,8 +986,11 @@ def st_download(options, args): object_queue = Queue(10000) def _download_object((container, obj), conn): try: - content_type, content_length, _, etag, metadata, body = \ + headers, body = \ conn.get_object(container, obj, resp_chunk_size=65536) + content_type = headers.get('content-type') + content_length = int(headers.get('content-length')) + etag = headers.get('etag') path = options.yes_all and join(container, obj) or obj if path[:1] in ('/', '\\'): path = path[1:] @@ -925,8 +1021,8 @@ def st_download(options, args): options.error_queue.put( '%s: read_length != content_length, %d != %d' % (path, read_length, content_length)) - if 'mtime' in metadata: - mtime = float(metadata['mtime']) + if 'x-object-meta-mtime' in headers: + mtime = float(headers['x-object-meta-mtime']) utime(path, (mtime, mtime)) if options.verbose: options.print_queue.put(path) @@ -941,7 +1037,7 @@ def st_download(options, args): marker = '' while True: objects = [o['name'] for o in - conn.get_container(container, marker=marker)] + conn.get_container(container, marker=marker)[1]] if not objects: break for obj in objects: @@ -969,7 +1065,7 @@ def st_download(options, args): marker = '' while True: containers = [c['name'] - for c in conn.get_account(marker=marker)] + for c in conn.get_account(marker=marker)[1]] if not containers: break for container in containers: @@ -1016,10 +1112,11 @@ def st_list(options, args): marker = '' while True: if not args: - items = conn.get_account(marker=marker, prefix=options.prefix) + items = \ + conn.get_account(marker=marker, prefix=options.prefix)[1] else: items = conn.get_container(args[0], marker=marker, - prefix=options.prefix, delimiter=options.delimiter) + prefix=options.prefix, delimiter=options.delimiter)[1] if not items: break for item in items: @@ -1042,46 +1139,85 @@ def st_stat(options, args): conn = Connection(options.auth, options.user, options.key) if not args: try: - container_count, object_count, bytes_used = conn.head_account() + headers = conn.head_account() + container_count = int(headers.get('x-account-container-count', 0)) + object_count = int(headers.get('x-account-object-count', 0)) + bytes_used = int(headers.get('x-account-bytes-used', 0)) options.print_queue.put(''' Account: %s Containers: %d Objects: %d Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], container_count, object_count, bytes_used)) + for key, value in headers.items(): + if key.startswith('x-account-meta-'): + options.print_queue.put('%10s: %s' % ('Meta %s' % + key[len('x-account-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-account-meta-') and key not in ( + 'content-length', 'date', 'x-account-container-count', + 'x-account-object-count', 'x-account-bytes-used'): + options.print_queue.put( + '%10s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise options.error_queue.put('Account not found') elif len(args) == 1: try: - object_count, bytes_used = conn.head_container(args[0]) + headers = conn.head_container(args[0]) + object_count = int(headers.get('x-container-object-count', 0)) + bytes_used = int(headers.get('x-container-bytes-used', 0)) options.print_queue.put(''' Account: %s Container: %s Objects: %d - Bytes: %d'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], - object_count, bytes_used)) + Bytes: %d + Read ACL: %s +Write ACL: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], + object_count, bytes_used, + headers.get('x-container-read', ''), + headers.get('x-container-write', ''))) + for key, value in headers.items(): + if key.startswith('x-container-meta-'): + options.print_queue.put('%9s: %s' % ('Meta %s' % + key[len('x-container-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-container-meta-') and key not in ( + 'content-length', 'date', 'x-container-object-count', + 'x-container-bytes-used', 'x-container-read', + 'x-container-write'): + options.print_queue.put( + '%9s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise options.error_queue.put('Container %s not found' % repr(args[0])) elif len(args) == 2: try: - content_type, content_length, last_modified, etag, metadata = \ - conn.head_object(args[0], args[1]) + headers = conn.head_object(args[0], args[1]) options.print_queue.put(''' Account: %s Container: %s Object: %s Content Type: %s -Content Length: %d +Content Length: %s Last Modified: %s ETag: %s'''.strip('\n') % (conn.url.rsplit('/', 1)[-1], args[0], - args[1], content_type, content_length, - last_modified, etag)) - for key, value in metadata.items(): - options.print_queue.put('%14s: %s' % ('Meta %s' % key, value)) + args[1], headers.get('content-type'), + headers.get('content-length'), + headers.get('last-modified'), + headers.get('etag'))) + for key, value in headers.items(): + if key.startswith('x-object-meta-'): + options.print_queue.put('%14s: %s' % ('Meta %s' % + key[len('x-object-meta-'):].title(), value)) + for key, value in headers.items(): + if not key.startswith('x-object-meta-') and key not in ( + 'content-type', 'content-length', 'last-modified', + 'etag', 'date'): + options.print_queue.put( + '%14s: %s' % (key.title(), value)) except ClientException, err: if err.http_status != 404: raise @@ -1092,6 +1228,63 @@ Content Length: %d (basename(argv[0]), st_stat_help)) +st_post_help = ''' +post [options] [container] [object] + Updates meta information for the account, container, or object depending on + the args given. If the container is not found, it will be created + automatically; but this is not true for accounts and objects. Containers + also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m + or --meta option is allowed on all and used to define the user meta data + items to set in the form Name:Value. This option can be repeated. Example: + post -m Color:Blue -m Size:Large'''.strip('\n') +def st_post(options, args): + conn = Connection(options.auth, options.user, options.key) + if not args: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Account-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + try: + conn.post_account(headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Account not found') + elif len(args) == 1: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Container-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + if options.read_acl is not None: + headers['X-Container-Read'] = options.read_acl + if options.write_acl is not None: + headers['X-Container-Write'] = options.write_acl + try: + conn.post_container(args[0], headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + conn.put_container(args[0], headers=headers) + elif len(args) == 2: + headers = {} + for item in options.meta: + split_item = item.split(':') + headers['X-Object-Meta-' + split_item[0]] = \ + len(split_item) > 1 and split_item[1] + try: + conn.post_object(args[0], args[1], headers=headers) + except ClientException, err: + if err.http_status != 404: + raise + options.error_queue.put('Object %s not found' % + repr('%s/%s' % (args[0], args[1]))) + else: + options.error_queue.put('Usage: %s [options] %s' % + (basename(argv[0]), st_post_help)) + + st_upload_help = ''' upload [options] container file_or_directory [file_or_directory] [...] Uploads to the given container the files and directories specified by the @@ -1108,35 +1301,41 @@ def st_upload(options, args): obj = path if obj.startswith('./') or obj.startswith('.\\'): obj = obj[2:] - metadata = {'mtime': str(getmtime(path))} + put_headers = {'x-object-meta-mtime': str(getmtime(path))} if dir_marker: if options.changed: try: - ct, cl, lm, et, md = conn.head_object(args[0], obj) + headers = conn.head_object(args[0], obj) + ct = headers.get('content-type') + cl = int(headers.get('content-length')) + et = headers.get('etag') + mt = headers.get('x-object-meta-mtime') if ct.split(';', 1)[0] == 'text/directory' and \ cl == 0 and \ et == 'd41d8cd98f00b204e9800998ecf8427e' and \ - md.get('mtime') == metadata['mtime']: + mt == put_headers['x-object-meta-mtime']: return except ClientException, err: if err.http_status != 404: raise conn.put_object(args[0], obj, '', content_length=0, content_type='text/directory', - metadata=metadata) + headers=put_headers) else: if options.changed: try: - ct, cl, lm, et, md = conn.head_object(args[0], obj) + headers = conn.head_object(args[0], obj) + cl = int(headers.get('content-length')) + mt = headers.get('x-object-meta-mtime') if cl == getsize(path) and \ - md.get('mtime') == metadata['mtime']: + mt == put_headers['x-object-meta-mtime']: return except ClientException, err: if err.http_status != 404: raise conn.put_object(args[0], obj, open(path, 'rb'), content_length=getsize(path), - metadata=metadata) + headers=put_headers) if options.verbose: options.print_queue.put(obj) except OSError, err: @@ -1190,6 +1389,7 @@ Commands: %(st_stat_help)s %(st_list_help)s %(st_upload_help)s + %(st_post_help)s %(st_download_help)s %(st_delete_help)s @@ -1215,6 +1415,18 @@ Example: help='For the list command on containers: will roll up ' 'items with the given delimiter (see Cloud Files ' 'general documentation for what this means).') + parser.add_option('-r', '--read-acl', dest='read_acl', + help='Sets the Read ACL with post container commands. ' + 'Quick summary of ACL syntax: .ref:any, ' + '.ref:-.example.com, .ref:www.example.com, account1, ' + 'account2:user2') + parser.add_option('-w', '--write-acl', dest='write_acl', + help='Sets the Write ACL with post container commands. ' + 'Quick summary of ACL syntax: account1, account2:user2') + parser.add_option('-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item of the syntax name:value ' + 'for use with post commands. This option may be ' + 'repeated. Example: -m Color:Blue -m Size:Large') parser.add_option('-A', '--auth', dest='auth', help='URL for obtaining an auth token') parser.add_option('-U', '--user', dest='user', @@ -1235,7 +1447,7 @@ overridden with -A, -U, or -K.'''.strip('\n') if not getattr(options, attr, None): exit(required_help) - commands = ('delete', 'download', 'list', 'stat', 'upload') + commands = ('delete', 'download', 'list', 'post', 'stat', 'upload') if not args or args[0] not in commands: parser.print_usage() if args: diff --git a/bin/swift-stats-populate b/bin/swift-stats-populate index 793c8b6ce4..8ea210cb65 100755 --- a/bin/swift-stats-populate +++ b/bin/swift-stats-populate @@ -48,7 +48,8 @@ def put_object(connpool, container, obj, report): global retries_done try: with connpool.item() as conn: - conn.put_object(container, obj, obj, metadata={'stats': obj}) + conn.put_object(container, obj, obj, + headers={'x-object-meta-stats': obj}) retries_done += conn.attempts - 1 if report: report(True) diff --git a/bin/swift-stats-report b/bin/swift-stats-report index 537a731697..bbb4410d2b 100755 --- a/bin/swift-stats-report +++ b/bin/swift-stats-report @@ -57,7 +57,8 @@ def get_error_log(prefix): def audit(coropool, connpool, account, container_ring, object_ring, options): begun = time() with connpool.item() as conn: - estimated_items = [conn.head_account()[0]] + estimated_items = \ + [int(conn.head_account()['x-account-container-count'])] items_completed = [0] retries_done = [0] containers_missing_replicas = {} @@ -130,7 +131,8 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): cmarker = '' while True: with connpool.item() as conn: - containers = [c['name'] for c in conn.get_account(marker=cmarker)] + containers = \ + [c['name'] for c in conn.get_account(marker=cmarker)[1]] if not containers: break cmarker = containers[-1] @@ -142,7 +144,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): while True: with connpool.item() as conn: objects = [o['name'] for o in - conn.get_container(container, marker=omarker)] + conn.get_container(container, marker=omarker)[1]] if not objects: break omarker = objects[-1] @@ -183,7 +185,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring, with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_dispersion_', - full_listing=True)] + full_listing=True)[1]] containers_listed = len(containers) if not containers_listed: print >>stderr, 'No containers to query. Has stats-populate been run?' @@ -262,7 +264,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring, options): with connpool.item() as conn: try: objects = [o['name'] for o in conn.get_container(container, - prefix='stats_object_dispersion_', full_listing=True)] + prefix='stats_object_dispersion_', full_listing=True)[1]] except ClientException, err: if err.http_status != 404: raise @@ -384,7 +386,7 @@ def container_head_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def head(container): with connpool.item() as conn: @@ -425,7 +427,7 @@ def container_get_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def get(container): with connpool.item() as conn: @@ -463,7 +465,8 @@ def container_standard_listing_report(coropool, connpool, options): print 'Listing big_container', with connpool.item() as conn: try: - value = len(conn.get_container('big_container', full_listing=True)) + value = \ + len(conn.get_container('big_container', full_listing=True)[1]) except ClientException, err: if err.http_status != 404: raise @@ -486,7 +489,7 @@ def container_prefix_listing_report(coropool, connpool, options): try: for x in xrange(256): value += len(conn.get_container('big_container', - prefix=('%02x' % x), full_listing=True)) + prefix=('%02x' % x), full_listing=True)[1]) except ClientException, err: if err.http_status != 404: raise @@ -511,7 +514,7 @@ def container_prefix_delimiter_listing_report(coropool, connpool, options): try: with connpool.item() as conn: listing = conn.get_container('big_container', - marker=marker, prefix=prefix, delimiter='/') + marker=marker, prefix=prefix, delimiter='/')[1] except ClientException, err: if err.http_status != 404: raise @@ -552,7 +555,7 @@ def container_delete_report(coropool, connpool, options): with connpool.item() as conn: containers = [c['name'] for c in conn.get_account(prefix='stats_container_put_', - full_listing=True)] + full_listing=True)[1]] count = len(containers) def delete(container): with connpool.item() as conn: @@ -630,7 +633,7 @@ def object_head_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def head(obj): with connpool.item() as conn: @@ -670,7 +673,7 @@ def object_get_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def get(obj): with connpool.item() as conn: @@ -710,7 +713,7 @@ def object_delete_report(coropool, connpool, options): next_report = [time() + 2] with connpool.item() as conn: objects = [o['name'] for o in conn.get_container('stats_object_put', - prefix='stats_object_put_', full_listing=True)] + prefix='stats_object_put_', full_listing=True)[1]] count = len(objects) def delete(obj): with connpool.item() as conn: @@ -791,7 +794,7 @@ Usage: %prog [options] [conf_file] else: options.retries = int(conf.get('retries', 5)) if not options.csv_output: - csv_output = conf.get('csv_output', '/etc/swift/stats.csv') + options.csv_output = conf.get('csv_output', '/etc/swift/stats.csv') coropool = GreenPool(size=concurrency) @@ -908,8 +911,8 @@ Usage: %prog [options] [conf_file] if options.csv_output != 'None': try: - if not os.path.exists(csv_output): - f = open(csv_output, 'wb') + if not os.path.exists(options.csv_output): + f = open(options.csv_output, 'wb') f.write('Timestamp,' 'Container Dispersion Report Time,' 'Container Dispersion Report Value,' @@ -936,7 +939,7 @@ Usage: %prog [options] [conf_file] 'Object DELETE Report Success Rate\r\n') csv = csv.writer(f) else: - csv = csv.writer(open(csv_output, 'ab')) + csv = csv.writer(open(options.csv_output, 'ab')) csv.writerow(report) except Exception, err: print >>stderr, 'Could not write CSV report:', err diff --git a/swift/common/client.py b/swift/common/client.py index c9ff71e274..4832452130 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -194,18 +194,21 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of accounts + :returns: a tuple of (response headers, a list of containers) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_account(url, token, marker, limit, prefix, http_conn) + rv = get_account(url, token, marker, limit, prefix, http_conn) + listing = rv[1] while listing: - rv.extend(listing) marker = listing[-1]['name'] - listing = get_account(url, token, marker, limit, prefix, http_conn) + listing = \ + get_account(url, token, marker, limit, prefix, http_conn)[1] + if listing: + rv.extend(listing) return rv parsed, conn = http_conn qs = 'format=json' @@ -218,6 +221,9 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn.request('GET', '%s?%s' % (parsed.path, qs), '', {'X-Auth-Token': token}) resp = conn.getresponse() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status < 200 or resp.status >= 300: resp.read() raise ClientException('Account GET failed', http_scheme=parsed.scheme, @@ -226,8 +232,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, http_reason=resp.reason) if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_account(url, token, http_conn=None): @@ -238,7 +244,8 @@ def head_account(url, token, http_conn=None): :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (container count, object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -253,9 +260,36 @@ def head_account(url, token, http_conn=None): http_host=conn.host, http_port=conn.port, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-account-container-count', 0)), \ - int(resp.getheader('x-account-object-count', 0)), \ - int(resp.getheader('x-account-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers + + +def post_account(url, token, headers, http_conn=None): + """ + Update an account's metadata. + + :param url: storage URL + :param token: auth token + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + headers['X-Auth-Token'] = token + conn.request('POST', parsed.path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Account POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) def get_container(url, token, container, marker=None, limit=None, @@ -275,23 +309,25 @@ def get_container(url, token, container, marker=None, limit=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings - :returns: a list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if not http_conn: http_conn = http_connection(url) if full_listing: - rv = [] - listing = get_container(url, token, container, marker, limit, prefix, - delimiter, http_conn) + rv = get_container(url, token, container, marker, limit, prefix, + delimiter, http_conn) + listing = rv[1] while listing: - rv.extend(listing) if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, - prefix, delimiter, http_conn) + prefix, delimiter, http_conn)[1] + if listing: + rv[1].extend(listing) return rv parsed, conn = http_conn path = '%s/%s' % (parsed.path, quote(container)) @@ -312,10 +348,13 @@ def get_container(url, token, container, marker=None, limit=None, http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_query=qs, http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def head_container(url, token, container, http_conn=None): @@ -327,7 +366,8 @@ def head_container(url, token, container, http_conn=None): :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -343,17 +383,20 @@ def head_container(url, token, container, http_conn=None): http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count', 0)), \ - int(resp.getheader('x-container-bytes-used', 0)) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers -def put_container(url, token, container, http_conn=None): +def put_container(url, token, container, headers=None, http_conn=None): """ Create a container :param url: storage URL :param token: auth token :param container: container name to create + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP PUT request failed @@ -363,7 +406,10 @@ def put_container(url, token, container, http_conn=None): else: parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) - conn.request('PUT', path, '', {'X-Auth-Token': token}) + if not headers: + headers = {} + headers['X-Auth-Token'] = token + conn.request('PUT', path, '', headers) resp = conn.getresponse() resp.read() if resp.status < 200 or resp.status >= 300: @@ -373,6 +419,34 @@ def put_container(url, token, container, http_conn=None): http_reason=resp.reason) +def post_container(url, token, container, headers, http_conn=None): + """ + Update a container's metadata. + + :param url: storage URL + :param token: auth token + :param container: container name to update + :param headers: additional headers to include in the request + :param http_conn: HTTP connection object (If None, it will create the + conn object) + :raises ClientException: HTTP POST request failed + """ + if http_conn: + parsed, conn = http_conn + else: + parsed, conn = http_connection(url) + path = '%s/%s' % (parsed.path, quote(container)) + headers['X-Auth-Token'] = token + conn.request('POST', path, '', headers) + resp = conn.getresponse() + resp.read() + if resp.status < 200 or resp.status >= 300: + raise ClientException('Container POST failed', + http_scheme=parsed.scheme, http_host=conn.host, + http_port=conn.port, http_path=path, http_status=resp.status, + http_reason=resp.reason) + + def delete_container(url, token, container, http_conn=None): """ Delete a container @@ -410,8 +484,12 @@ def get_object(url, token, container, name, http_conn=None, :param name: object name to get :param http_conn: HTTP connection object (If None, it will create the conn object) - :param resp_chunk_size: if defined, chunk size of data to read - :returns: a list of objects + :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If + you specify a resp_chunk_size you must fully read + the object's contents before making another + request. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ if http_conn: @@ -440,12 +518,10 @@ def get_object(url, token, container, name, http_conn=None, object_body = _object_body() else: object_body = resp.read() - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata, \ - object_body + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def head_object(url, token, container, name, http_conn=None): @@ -458,8 +534,8 @@ def head_object(url, token, container, name, http_conn=None): :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) - :returns: a tuple of (content type, content length, last modfied, etag, - dictionary of metadata) + :returns: a dict containing the response's headers (all header names will + be lowercase) :raises ClientException: HTTP HEAD request failed """ if http_conn: @@ -478,16 +554,15 @@ def head_object(url, token, container, name, http_conn=None): for key, value in resp.getheaders(): if key.lower().startswith('x-object-meta-'): metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length', 0)), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers -def put_object(url, token, container, name, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None, http_conn=None): +def put_object(url, token, container, name, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, headers=None, + http_conn=None): """ Put an object @@ -495,12 +570,12 @@ def put_object(url, token, container, name, contents, metadata={}, :param token: auth token :param container: container name that the object is in :param name: object name to put - :param contents: file like object to read object data from - :param metadata: dictionary of object metadata + :param contents: a string or a file like object to read object data from :param content_length: value to send as content-length header :param etag: etag of contents :param chunk_size: chunk size of data to write :param content_type: value to send as content-type header + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :returns: etag from server response @@ -511,9 +586,9 @@ def put_object(url, token, container, name, contents, metadata={}, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + if not headers: + headers = {} + headers['X-Auth-Token'] = token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -549,7 +624,7 @@ def put_object(url, token, container, name, contents, metadata={}, return resp.getheader('etag').strip('"') -def post_object(url, token, container, name, metadata, http_conn=None): +def post_object(url, token, container, name, headers, http_conn=None): """ Change object metadata @@ -557,7 +632,7 @@ def post_object(url, token, container, name, metadata, http_conn=None): :param token: auth token :param container: container name that the object is in :param name: object name to change - :param metadata: dictionary of object metadata + :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) :raises ClientException: HTTP POST request failed @@ -567,9 +642,7 @@ def post_object(url, token, container, name, metadata, http_conn=None): else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers = {'X-Auth-Token': token} - for key, value in metadata.iteritems(): - headers['X-Object-Meta-%s' % quote(key)] = quote(value) + headers['X-Auth-Token'] = token conn.request('POST', path, '', headers) resp = conn.getresponse() resp.read() @@ -682,6 +755,10 @@ class Connection(object): return self._retry(get_account, marker=marker, limit=limit, prefix=prefix, full_listing=full_listing) + def post_account(self, headers): + """Wrapper for :func:`post_account`""" + return self._retry(post_account, headers) + def head_container(self, container): """Wrapper for :func:`head_container`""" return self._retry(head_container, container) @@ -696,9 +773,13 @@ class Connection(object): limit=limit, prefix=prefix, delimiter=delimiter, full_listing=full_listing) - def put_container(self, container): + def put_container(self, container, headers=None): """Wrapper for :func:`put_container`""" - return self._retry(put_container, container) + return self._retry(put_container, container, headers=headers) + + def post_container(self, container, headers): + """Wrapper for :func:`post_container`""" + return self._retry(post_container, container, headers) def delete_container(self, container): """Wrapper for :func:`delete_container`""" @@ -713,17 +794,17 @@ class Connection(object): return self._retry(get_object, container, obj, resp_chunk_size=resp_chunk_size) - def put_object(self, container, obj, contents, metadata={}, - content_length=None, etag=None, chunk_size=65536, - content_type=None): + def put_object(self, container, obj, contents, content_length=None, + etag=None, chunk_size=65536, content_type=None, + headers=None): """Wrapper for :func:`put_object`""" return self._retry(put_object, container, obj, contents, - metadata=metadata, content_length=content_length, etag=etag, - chunk_size=chunk_size, content_type=content_type) + content_length=content_length, etag=etag, chunk_size=chunk_size, + content_type=content_type, headers=headers) - def post_object(self, container, obj, metadata): + def post_object(self, container, obj, headers): """Wrapper for :func:`post_object`""" - return self._retry(post_object, container, obj, metadata) + return self._retry(post_object, container, obj, headers) def delete_object(self, container, obj): """Wrapper for :func:`delete_object`""" diff --git a/test/probe/test_account_failures.py b/test/probe/test_account_failures.py index 97397084e0..10eba803bc 100755 --- a/test/probe/test_account_failures.py +++ b/test/probe/test_account_failures.py @@ -39,8 +39,10 @@ class TestAccountFailures(unittest.TestCase): client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) - self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '0') + self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: @@ -56,8 +58,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') - self.assert_(client.head_account(self.url, self.token), (2, 0, 0)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '0') + self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: @@ -73,8 +77,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) get_to_final_state() - containers = client.get_account(self.url, self.token) - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -94,8 +100,10 @@ class TestAccountFailures(unittest.TestCase): client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -115,8 +123,10 @@ class TestAccountFailures(unittest.TestCase): 'once'])) for p in ps: p.wait() - self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) - containers = client.get_account(self.url, self.token) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '2') + self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: @@ -134,10 +144,12 @@ class TestAccountFailures(unittest.TestCase): '/etc/swift/account-server/%d.conf' % ((anodes[0]['port'] - 6002) / 10)]).pid sleep(2) - # This is the earlier object count and bytes because the first node - # doesn't have the newest udpates yet. - self.assert_(client.head_account(self.url, self.token), (2, 1, 4)) - containers = client.get_account(self.url, self.token) + # This is the earlier counts and bytes because the first node doesn't + # have the newest udpates yet. + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '2') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: @@ -155,8 +167,10 @@ class TestAccountFailures(unittest.TestCase): self.assert_(found2) get_to_final_state() - containers = client.get_account(self.url, self.token) - self.assert_(client.head_account(self.url, self.token), (2, 2, 9)) + headers, containers = client.get_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '2') + self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index 932f821c1c..9c497ca88b 100755 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -40,23 +40,23 @@ class TestContainerFailures(unittest.TestCase): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[0]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.pids[self.port2server[cnodes[0]['port']]] = \ Popen(['swift-container-server', @@ -64,11 +64,11 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[0]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -87,7 +87,7 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account @@ -95,44 +95,44 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_second_node_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) kill(self.pids[self.port2server[cnodes[1]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.pids[self.port2server[cnodes[1]['port']]] = \ Popen(['swift-container-server', @@ -140,9 +140,9 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[1]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -161,42 +161,42 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): @@ -204,9 +204,9 @@ class TestContainerFailures(unittest.TestCase): client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ @@ -215,11 +215,11 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -238,7 +238,7 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account @@ -246,35 +246,35 @@ class TestContainerFailures(unittest.TestCase): client.put_object(self.url, self.token, container, object2, 'test') # First node still doesn't know object1 was deleted yet; this is okay. self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) def test_last_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in (1, 2): @@ -282,9 +282,9 @@ class TestContainerFailures(unittest.TestCase): client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) for x in (1, 2): self.pids[self.port2server[cnodes[x]['port']]] = \ @@ -293,9 +293,9 @@ class TestContainerFailures(unittest.TestCase): ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got @@ -314,29 +314,29 @@ class TestContainerFailures(unittest.TestCase): # account server, this'll pass, otherwise the first account server will # serve the listing and not have the container. # self.assert_(container in [c['name'] for c in - # client.get_account(self.url, self.token)]) + # client.get_account(self.url, self.token)[1]]) object2 = 'object2' # This will work because at least one (in this case, just one) account # server has to indicate the container exists for the put to continue. client.put_object(self.url, self.token, container, object2, 'test') self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) # And, of course, our new object2 exists. self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) get_to_final_state() # Our container delete never "finalized" because we started using it # before the delete settled. self.assert_(container in [c['name'] for c in - client.get_account(self.url, self.token)]) + client.get_account(self.url, self.token)[1]]) # And, so our object2 should still exist and object1's delete should # have finalized. self.assert_(object1 not in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) self.assert_(object2 in [o['name'] for o in - client.get_container(self.url, self.token, container)]) + client.get_container(self.url, self.token, container)[1]]) if __name__ == '__main__': diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 20a29772c4..37767cbb5e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -75,7 +75,7 @@ class TestObjectHandoff(unittest.TestCase): raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in - client.get_container(self.url, self.token, container)] + client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: @@ -126,10 +126,9 @@ class TestObjectHandoff(unittest.TestCase): kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.post_object(self.url, self.token, container, obj, - {'probe': 'value'}) - ometadata = client.head_object( - self.url, self.token, container, obj)[-1] - if ometadata.get('probe') != 'value': + headers={'x-object-meta-probe': 'value'}) + ometadata = client.head_object(self.url, self.token, container, obj) + if ometadata.get('x-object-meta-probe') != 'value': raise Exception('Metadata incorrect, was %s' % repr(ometadata)) exc = False try: @@ -177,7 +176,7 @@ class TestObjectHandoff(unittest.TestCase): if not exc: raise Exception('Regular object HEAD was still successful') objs = [o['name'] for o in - client.get_container(self.url, self.token, container)] + client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: diff --git a/test/probe/test_running_with_each_type_down.py b/test/probe/test_running_with_each_type_down.py index fada1b18ac..7f2352d6ce 100755 --- a/test/probe/test_running_with_each_type_down.py +++ b/test/probe/test_running_with_each_type_down.py @@ -56,16 +56,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase): pass client.put_object(self.url, self.token, 'container1', 'object1', '1234') get_to_final_state() - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) @@ -84,15 +87,18 @@ class TestRunningWithEachTypeDown(unittest.TestCase): '/etc/swift/object-server/%d.conf' % ((onodes[0]['port'] - 6000) / 10)]).pid sleep(2) - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True # The account node was previously down. self.assert_(not found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) @@ -101,16 +107,19 @@ class TestRunningWithEachTypeDown(unittest.TestCase): self.assert_(found1) get_to_final_state() - self.assert_(client.head_account(self.url, self.token), (1, 1, 1234)) + headers, containers = client.head_account(self.url, self.token) + self.assertEquals(headers['x-account-container-count'], '1') + self.assertEquals(headers['x-account-object-count'], '1') + self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False - for container in client.get_account(self.url, self.token): + for container in containers: if container['name'] == 'container1': found1 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) found1 = False - for obj in client.get_container(self.url, self.token, 'container1'): + for obj in client.get_container(self.url, self.token, 'container1')[1]: if obj['name'] == 'object1': found1 = True self.assertEquals(obj['bytes'], 4) diff --git a/test/unit/common/test_client.py b/test/unit/common/test_client.py index 1f67f3e020..23d5c6d848 100644 --- a/test/unit/common/test_client.py +++ b/test/unit/common/test_client.py @@ -191,7 +191,7 @@ class TestGetAccount(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_account('http://www.test.com', 'asdf') + value = c.get_account('http://www.test.com', 'asdf')[1] self.assertEquals(value, []) @@ -200,7 +200,10 @@ class TestHeadAccount(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.head_account('http://www.tests.com', 'asdf') - self.assertEquals(value, (0, 0, 0)) + # TODO: Hmm. This doesn't really test too much as it uses a fake that + # always returns the same dict. I guess it "exercises" the code, so + # I'll leave it for now. + self.assertEquals(type(value), dict) def test_server_error(self): c.http_connection = self.fake_http_connection(500) @@ -212,7 +215,7 @@ class TestGetContainer(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_container('http://www.test.com', 'asdf', 'asdf') + value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1] self.assertEquals(value, []) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 20b9f95fec..0db4e6df47 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -82,7 +82,7 @@ def fake_http_connect(*code_iter, **kwargs): pass if 'slow' in kwargs: headers['content-length'] = '4' - return headers + return headers.items() def read(self, amt=None): if 'slow' in kwargs: if self.sent < 4: @@ -98,7 +98,7 @@ def fake_http_connect(*code_iter, **kwargs): self.received += 1 sleep(0.1) def getheader(self, name, default=None): - return self.getheaders().get(name.lower(), default) + return dict(self.getheaders()).get(name.lower(), default) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) if not isinstance(x, (tuple, list)): From 65eb19f103df7df89c12556bbee71fe04e035ef3 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 3 Sep 2010 22:33:41 -0700 Subject: [PATCH 06/16] Documentation of the new auth and acls middleware modules and bugfixes --- doc/source/misc.rst | 9 +++ swift/common/middleware/acl.py | 120 ++++++++++++++++++++++++++------ swift/common/middleware/auth.py | 17 ++++- 3 files changed, 124 insertions(+), 22 deletions(-) diff --git a/doc/source/misc.rst b/doc/source/misc.rst index bae61a699a..cfd188e65a 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -42,6 +42,15 @@ Auth :members: :show-inheritance: +.. _acls: + +ACLs +==== + +.. automodule:: swift.common.middleware.acl + :members: + :show-inheritance: + .. _wsgi: WSGI diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index bfa88b8105..bf3148a16a 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -14,37 +14,101 @@ # limitations under the License. def clean_acl(name, value): + """ + Returns a cleaned ACL header value, validating that it meets the formatting + requirements for standard Swift ACL strings. + + The ACL format is:: + + [item[,item...]] + + Each item can be a group name to give access to or a referrer designation + to grant or deny based on the HTTP Referer header. + + The referrer designation format is:: + + .ref:[-]value + + The value can be "any" to specify any referrer host is allowed access, a + specific host name like "www.example.com", or if it has a leading period + "." it is a domain name specification, like ".example.com". The leading + minus sign "-" indicates referrer hosts that should be denied access. + + Referrer access is applied in the order they are specified. For example, + .ref:.example.com,.ref:-thief.example.com would allow all hosts ending with + .example.com except for the specific host thief.example.com. + + Example valid ACLs:: + + .ref:any + .ref:any,.ref:-.thief.com + .ref:any,.ref:-.thief.com,bobs_account,sues_account:sue + bobs_account,sues_account:sue + + Example invalid ACLs:: + + .ref: + .ref:- + + Also, .ref designations aren't allowed in headers whose names include the + word 'write'. + + ACLs that are "messy" will be cleaned up. Examples: + + ====================== ====================== + Original Cleaned + ---------------------- ---------------------- + bob, sue bob,sue + bob , sue bob,sue + bob,,,sue bob,sue + .ref : any .ref:any + ====================== ====================== + + :param name: The name of the header being cleaned, such as X-Container-Read + or X-Container-Write. + :param value: The value of the header being cleaned. + :returns: The value, cleaned of extraneous formatting. + :raises ValueError: If the value does not meet the ACL formatting + requirements; the error message will indicate why. + """ values = [] for raw_value in value.lower().split(','): raw_value = raw_value.strip() if raw_value: - if ':' in raw_value: + if ':' not in raw_value: + values.append(raw_value) + else: first, second = (v.strip() for v in raw_value.split(':', 1)) - if not first: - raise ValueError('No value before colon in %s' % - repr(raw_value)) - if first == '.ref' and 'write' in name: + if first != '.ref': + values.append(raw_value) + elif 'write' in name: raise ValueError('Referrers not allowed in write ACLs: %s' % repr(raw_value)) - if second: - if first == '.ref' and second[0] == '-': + elif not second: + raise ValueError('No value after referrer designation in ' + '%s' % repr(raw_value)) + else: + if second[0] == '-': second = second[1:].strip() if not second: raise ValueError('No value after referrer deny ' 'designation in %s' % repr(raw_value)) second = '-' + second values.append('%s:%s' % (first, second)) - elif first == '.ref': - raise ValueError('No value after referrer designation in ' - '%s' % repr(raw_value)) - else: - values.append(first) - else: - values.append(raw_value) return ','.join(values) def parse_acl(acl_string): + """ + Parses a standard Swift ACL string into a referrers list and groups list. + + See :func:`clean_acl` for documentation of the standard Swift ACL format. + + :param acl_string: The standard Swift ACL string to parse. + :returns: A tuple of (referrers, groups) where referrers is a list of + referrer designations (without the leading .ref:) and groups is a + list of groups to allow access. + """ referrers = [] groups = [] if acl_string: @@ -56,15 +120,29 @@ def parse_acl(acl_string): return referrers, groups -def referrer_allowed(req, referrers): +def referrer_allowed(referrer, referrer_acl): + """ + Returns True if the referrer should be allowed based on the referrer_acl + list (as returned by :func:`parse_acl`). + + See :func:`clean_acl` for documentation of the standard Swift ACL format. + + :param referrer: The value of the HTTP Referer header. + :param referrer_acl: The list of referrer designations as returned by + :func:`parse_acl`. + :returns: True if the referrer should be allowed; False if not. + """ allow = False - if referrers: - parts = req.referer.split('//', 1) - if len(parts) == 2: - rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() - else: + if referrer_acl: + if not referrer: rhost = 'unknown' - for mhost in referrers: + else: + parts = referrer.split('//', 1) + if len(parts) == 2: + rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() + else: + rhost = 'unknown' + for mhost in referrer_acl: if mhost[0] == '-': mhost = mhost[1:] if mhost == rhost or \ diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index aee5e99353..e433c7fda0 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -24,6 +24,7 @@ from swift.common.utils import cache_from_env, split_path class DevAuth(object): + """Auth Middleware that uses the dev auth server.""" def __init__(self, app, conf): self.app = app @@ -35,6 +36,11 @@ class DevAuth(object): self.timeout = int(conf.get('node_timeout', 10)) def __call__(self, env, start_response): + """ + Accepts a standard WSGI application call, authenticating the request + and installing callback hooks for authorization and ACL header + validation. + """ user = None token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) if token: @@ -64,13 +70,17 @@ class DevAuth(object): return self.app(env, start_response) def authorize(self, req): + """ + Returns None if the request is authorized to continue or a standard + WSGI response callable if not. + """ version, account, container, obj = split_path(req.path, 1, 4, True) if not account: return self.denied_response(req) if req.remote_user and account in req.remote_user.split(','): return None referrers, groups = parse_acl(getattr(req, 'acl', None)) - if referrer_allowed(req, referrers): + if referrer_allowed(req.referer, referrers): return None if not req.remote_user: return self.denied_response(req) @@ -80,6 +90,10 @@ class DevAuth(object): return self.denied_response(req) def denied_response(self, req): + """ + Returns a standard WSGI response callable with the status of 403 or 401 + depending on whether the REMOTE_USER is set or not. + """ if req.remote_user: return HTTPForbidden(request=req) else: @@ -87,6 +101,7 @@ class DevAuth(object): def filter_factory(global_conf, **local_conf): + """Returns a WSGI filter app for use with paste.deploy.""" conf = global_conf.copy() conf.update(local_conf) def auth_filter(app): From 0066ed02d7c1dc097b2de887299bb0130cac7796 Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Sep 2010 19:21:08 -0700 Subject: [PATCH 07/16] Per Chuck's suggestion, changed noaccess to admin access, where admin access is not the default. Also, changed swift-auth-create-account to swift-auth-add-user with changes to use optparse --- ...uth-create-account => swift-auth-add-user} | 57 +++++++--------- doc/source/development_saio.rst | 13 ++-- doc/source/howto_cyberduck.rst | 4 +- setup.py | 2 +- swift/auth/server.py | 66 +++++++++++-------- test/functional/sample.conf | 6 +- test/functionalnosetests/test_container.py | 2 +- 7 files changed, 76 insertions(+), 74 deletions(-) rename bin/{swift-auth-create-account => swift-auth-add-user} (50%) diff --git a/bin/swift-auth-create-account b/bin/swift-auth-add-user similarity index 50% rename from bin/swift-auth-create-account rename to bin/swift-auth-add-user index a96e175529..76c1be6ac3 100755 --- a/bin/swift-auth-create-account +++ b/bin/swift-auth-add-user @@ -15,6 +15,7 @@ # limitations under the License. from ConfigParser import ConfigParser +from optparse import OptionParser from os.path import basename from sys import argv, exit @@ -22,45 +23,37 @@ from swift.common.bufferedhttp import http_connect_raw as http_connect if __name__ == '__main__': - f = '/etc/swift/auth-server.conf' - good = False - noaccess = False - if len(argv) == 6 and argv[4] == 'noaccess': - good = True - noaccess = True - f = argv[5] - elif len(argv) == 5: - good = True - if argv[4] == 'noaccess': - noaccess = True - else: - f = argv[4] - elif len(argv) == 4: - good = True - if not good: - exit(''' -Syntax: %s [noaccess] [conf_file] -The noaccess keyword will create a user with no access to the account; another -user for the account will have to add the user to the ACLs for a container to -grant some access. - '''.strip() % basename(argv[0])) - new_account = argv[1] - new_user = argv[2] - new_password = argv[3] + default_conf = '/etc/swift/auth-server.conf' + parser = OptionParser( + usage='Usage: %prog [options] ') + parser.add_option('-c', '--conf', dest='conf', default=default_conf, + help='Configuration file to determine how to connect to the local ' + 'auth server (default: %s).' % default_conf) + parser.add_option('-a', '--admin', dest='admin', action='store_true', + default=False, help='Give the user administrator access; otherwise ' + 'the user will only have access to container specifically allowed ' + 'with ACLs.') + args = argv[1:] + if not args: + args.append('-h') + (options, args) = parser.parse_args(args) + if len(args) != 3: + parser.parse_args(['-h']) + account, user, password = args c = ConfigParser() - if not c.read(f): - exit('Unable to read conf file: %s' % f) + if not c.read(options.conf): + exit('Unable to read conf file: %s' % options.conf) conf = dict(c.items('app:auth-server')) host = conf.get('bind_ip', '127.0.0.1') port = int(conf.get('bind_port', 11000)) ssl = conf.get('cert_file') is not None - path = '/account/%s/%s' % (new_account, new_user) - headers = {'X-Auth-Key': new_password} - if noaccess: - headers['X-User-No-Access'] = 'true' + path = '/account/%s/%s' % (account, user) + headers = {'X-Auth-User-Key': password} + if options.admin: + headers['X-Auth-User-Admin'] = 'true' conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl) resp = conn.getresponse() if resp.status == 204: print resp.getheader('x-storage-url') else: - print 'Account creation failed. (%d)' % resp.status + print 'Update failed: %s %s' % (resp.status, resp.reason) diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 1c0ec8f84f..6725121bca 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -526,19 +526,20 @@ good idea what to do on other environments. #. `remakerings` #. `cd ~/swift/trunk; ./.unittests` #. `startmain` (The ``Unable to increase file descriptor limit. Running as non-root?`` warnings are expected and ok.) - #. `swift-auth-create-account test tester testing` + #. `swift-auth-add-user --admin test tester testing` #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` - #. `swift-auth-create-account test2 tester2 testing2` - #. `swift-auth-create-account test tester3 testing3 noaccess` + #. `swift-auth-add-user --admin test2 tester2 testing2` + #. `swift-auth-add-user test tester3 testing3` #. Create `/etc/swift/func_test.conf`:: cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf - #. `cd ~/swift/trunk; ./.functests` - #. `cd ~/swift/trunk; ./.probetests` (Note for future reference: probe tests - will reset your environment) + #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete + everything in the configured account.) + #. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your + environment as they call `resetswift` for each test.) If you plan to work on documentation (and who doesn't?!): diff --git a/doc/source/howto_cyberduck.rst b/doc/source/howto_cyberduck.rst index a372ebf1ab..be451f7d8c 100644 --- a/doc/source/howto_cyberduck.rst +++ b/doc/source/howto_cyberduck.rst @@ -107,9 +107,9 @@ Installing Swift For Use With Cyberduck cert_file = /etc/swift/cert.crt key_file = /etc/swift/cert.key -#. Use swift-auth-create-account to create a new account:: +#. Use swift-auth-add-user to create a new account and admin user:: - ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-create-account a3 b3 c3 + ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-add-user --admin a3 b3 c3 https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1/06228ccf-6d0a-4395-889e-e971e8de8781 .. note:: diff --git a/setup.py b/setup.py index 2a3084e110..ee8b7fed9d 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ setup( 'bin/st', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/swift-account-replicator', 'bin/swift-account-server', - 'bin/swift-auth-create-account', + 'bin/swift-auth-add-user', 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', 'bin/swift-container-auditor', 'bin/swift-container-replicator', diff --git a/swift/auth/server.py b/swift/auth/server.py index 39c8dd8837..2b9a6fe2c7 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -105,14 +105,14 @@ class AuthController(object): self.db_file = os.path.join(self.swift_dir, 'auth.db') self.conn = get_db_connection(self.db_file, okay_to_create=True) try: - self.conn.execute('SELECT noaccess FROM account LIMIT 1') + self.conn.execute('SELECT admin FROM account LIMIT 1') except sqlite3.OperationalError, err: - if str(err) == 'no such column: noaccess': + if str(err) == 'no such column: admin': self.conn.execute( - 'ALTER TABLE account ADD COLUMN noaccess TEXT') + "ALTER TABLE account ADD COLUMN admin TEXT DEFAULT 't'") self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT, noaccess TEXT)''') + user TEXT, password TEXT, admin TEXT)''') self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account ON account (account)''') try: @@ -224,7 +224,8 @@ class AuthController(object): Tests if the given token is a valid token :param token: The token to validate - :returns: (TTL, account, user, cfaccount) if valid, False otherwise + :returns: (TTL, account, user, cfaccount) if valid, False otherwise. + cfaccount will be None for users without admin access. """ begin = time() self.purge_old_tokens() @@ -248,7 +249,7 @@ class AuthController(object): return rv def create_account(self, new_account, new_user, new_password, - noaccess=False): + admin=False): """ Handles the create_account call for developers, used to request an account be created both on a Swift cluster and in the auth server @@ -266,9 +267,9 @@ class AuthController(object): :param new_account: The name for the new account :param new_user: The name for the new user :param new_password: The password for the new account - :param noaccess: If true, the user will be granted no access to the - account by default; another user will have to add the - user to the ACLs for containers to grant access. + :param admin: If true, the user will be granted full access to the + account; otherwise, another user will have to add the + user to the ACLs for containers to grant access. :returns: False if the create fails, 'already exists' if the user already exists, or storage url if successful @@ -283,7 +284,7 @@ class AuthController(object): if row: self.logger.info( 'ALREADY EXISTS create_account(%s, %s, _, %s) [%.02f]' % - (repr(new_account), repr(new_user), repr(noaccess), + (repr(new_account), repr(new_user), repr(admin), time() - begin)) return 'already exists' row = conn.execute( @@ -297,19 +298,19 @@ class AuthController(object): if not account_hash: self.logger.info( 'FAILED create_account(%s, %s, _, %s) [%.02f]' % - (repr(new_account), repr(new_user), repr(noaccess), + (repr(new_account), repr(new_user), repr(admin), time() - begin)) return False url = self.default_cluster_url.rstrip('/') + '/' + account_hash conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password, noaccess) + (account, url, cfaccount, user, password, admin) VALUES (?, ?, ?, ?, ?, ?)''', (new_account, url, account_hash, new_user, new_password, - noaccess and 't' or '')) + admin and 't' or '')) conn.commit() self.logger.info( 'SUCCESS create_account(%s, %s, _, %s) = %s [%.02f]' % - (repr(new_account), repr(new_user), repr(noaccess), repr(url), + (repr(new_account), repr(new_user), repr(admin), repr(url), time() - begin)) return url @@ -350,25 +351,31 @@ class AuthController(object): _, token = split_path(request.path, minsegs=2) except ValueError: return HTTPBadRequest() + # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise validation = self.validate_token(token) if not validation: return HTTPNotFound() - # X-Auth-User: account:user,account,cfaccount + groups = ['%s:%s' % (validation[1], validation[2]), validation[1]] + if validation[3]: # admin access to a cfaccount + groups.append(validation[3]) return HTTPNoContent(headers={'X-Auth-TTL': validation[0], - 'X-Auth-User': '%s:%s,%s,%s' % - (validation[1], validation[2], validation[1], validation[3])}) + 'X-Auth-User': ','.join(groups)}) - def handle_account_create(self, request): + def handle_add_user(self, request): """ - Handles Rest requests from developers to have an account created. + Handles Rest requests from developers to have a user added. If the + account specified doesn't exist, it will also be added. Currently, + updating a user's information (password, admin access) must be done by + directly updating the sqlite database. Valid URL paths: * PUT /account// - create the account Valid headers: - * X-Auth-Key: (Only required when creating an account) + * X-Auth-User-Key: + * X-Auth-User-Admin: - If the HTTP request returns with a 204, then the account was created, + If the HTTP request returns with a 204, then the user was added, and the storage url will be available in the X-Storage-Url header. :param request: webob.Request object @@ -377,11 +384,11 @@ class AuthController(object): _, account_name, user_name = split_path(request.path, minsegs=3) except ValueError: return HTTPBadRequest() - if 'X-Auth-Key' not in request.headers: - return HTTPBadRequest('X-Auth-Key is required') - password = request.headers['x-auth-key'] + if 'X-Auth-User-Key' not in request.headers: + return HTTPBadRequest('X-Auth-User-Key is required') + password = request.headers['x-auth-user-key'] storage_url = self.create_account(account_name, user_name, password, - request.headers.get('x-user-no-access')) + request.headers.get('x-auth-user-admin') == 'true') if storage_url == 'already exists': return HTTPBadRequest(storage_url) if not storage_url: @@ -458,13 +465,14 @@ class AuthController(object): self.purge_old_tokens() with self.get_conn() as conn: row = conn.execute(''' - SELECT cfaccount, url, noaccess FROM account + SELECT cfaccount, url, admin FROM account WHERE account = ? AND user = ? AND password = ?''', (account, user, password)).fetchone() if row is None: return HTTPUnauthorized() - cfaccount = row[2] and '.none' or row[0] + cfaccount = row[0] url = row[1] + admin = row[2] == 't' row = conn.execute(''' SELECT token FROM token WHERE account = ? AND user = ?''', (account, user)).fetchone() @@ -476,7 +484,7 @@ class AuthController(object): INSERT INTO token (token, created, account, user, cfaccount) VALUES (?, ?, ?, ?, ?)''', - (token, time(), account, user, cfaccount)) + (token, time(), account, user, admin and cfaccount or '')) conn.commit() return HTTPNoContent(headers={'x-auth-token': token, 'x-storage-token': token, @@ -503,7 +511,7 @@ class AuthController(object): elif req.method == 'GET' and req.path.startswith('/token/'): handler = self.handle_token elif req.method == 'PUT' and req.path.startswith('/account/'): - handler = self.handle_account_create + handler = self.handle_add_user elif req.method == 'POST' and \ req.path == '/recreate_accounts': handler = self.handle_account_recreate diff --git a/test/functional/sample.conf b/test/functional/sample.conf index b43e106bd8..983f2cf768 100644 --- a/test/functional/sample.conf +++ b/test/functional/sample.conf @@ -3,17 +3,17 @@ auth_host = 127.0.0.1 auth_port = 11000 auth_ssl = no -# Primary functional test account +# Primary functional test account (needs admin access to the account) account = test username = tester password = testing -# User on a second account +# User on a second account (needs admin access to the account) account2 = test2 username2 = tester2 password2 = testing2 -# User on same account as first, but with noaccess +# User on same account as first, but without admin access username3 = tester3 password3 = testing3 diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py index abda0f3188..6498f56423 100755 --- a/test/functionalnosetests/test_container.py +++ b/test/functionalnosetests/test_container.py @@ -462,7 +462,7 @@ class TestContainer(unittest.TestCase): resp.read() self.assertEquals(resp.status, 201) - def test_noaccess_user(self): + def test_nonadmin_user(self): if skip or skip3: raise SkipTest # Obtain the first account's string From 235c0e9bd53f25960768c9d352b6e29c647f9b6a Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Sep 2010 19:53:08 -0700 Subject: [PATCH 08/16] Fixed typos; bug fix in auth db upgrade; renamed a couple things to better reflect their new usage; updated docs --- bin/swift-auth-add-user | 2 +- doc/source/development_saio.rst | 7 +-- swift/auth/server.py | 63 +++++++++++++------------ swift/common/middleware/auth.py | 17 +++---- test/unit/auth/test_server.py | 82 ++++++++++++++++----------------- 5 files changed, 84 insertions(+), 87 deletions(-) diff --git a/bin/swift-auth-add-user b/bin/swift-auth-add-user index 76c1be6ac3..0f7eb3c3a2 100755 --- a/bin/swift-auth-add-user +++ b/bin/swift-auth-add-user @@ -31,7 +31,7 @@ if __name__ == '__main__': 'auth server (default: %s).' % default_conf) parser.add_option('-a', '--admin', dest='admin', action='store_true', default=False, help='Give the user administrator access; otherwise ' - 'the user will only have access to container specifically allowed ' + 'the user will only have access to containers specifically allowed ' 'with ACLs.') args = argv[1:] if not args: diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index 6725121bca..b583641ab5 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -532,12 +532,9 @@ good idea what to do on other environments. #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` #. `swift-auth-add-user --admin test2 tester2 testing2` #. `swift-auth-add-user test tester3 testing3` - #. Create `/etc/swift/func_test.conf`:: - - cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf - + #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete - everything in the configured account.) + everything in the configured accounts.) #. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your environment as they call `resetswift` for each test.) diff --git a/swift/auth/server.py b/swift/auth/server.py index 2b9a6fe2c7..e047397072 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -108,8 +108,8 @@ class AuthController(object): self.conn.execute('SELECT admin FROM account LIMIT 1') except sqlite3.OperationalError, err: if str(err) == 'no such column: admin': - self.conn.execute( - "ALTER TABLE account ADD COLUMN admin TEXT DEFAULT 't'") + self.conn.execute("ALTER TABLE account ADD COLUMN admin TEXT") + self.conn.execute("UPDATE account SET admin = 't'") self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( account TEXT, url TEXT, cfaccount TEXT, user TEXT, password TEXT, admin TEXT)''') @@ -248,25 +248,22 @@ class AuthController(object): (repr(token), repr(rv), time() - begin)) return rv - def create_account(self, new_account, new_user, new_password, - admin=False): + def create_user(self, account, user, password, admin=False): """ - Handles the create_account call for developers, used to request - an account be created both on a Swift cluster and in the auth server - database. + Handles the create_user call for developers, used to request a user be + added in the auth server database. If the account does not yet exist, + it will be created on the Swift cluster and the details recorded in the + auth server database. - This will make ReST requests to the Swift cluster's account servers - to have an account created on its side. The resulting account hash - along with the URL to use to access the account, the account name, the - user name, and the password is recorded in the auth server's database. - The url is constructed now and stored separately to support changing - the configuration file's default_cluster_url for directing new accounts - to a different Swift cluster while still supporting old accounts going - to the Swift clusters they were created on. + The url for the storage account is constructed now and stored + separately to support changing the configuration file's + default_cluster_url for directing new accounts to a different Swift + cluster while still supporting old accounts going to the Swift clusters + they were created on. - :param new_account: The name for the new account - :param new_user: The name for the new user - :param new_password: The password for the new account + :param account: The name for the new account + :param user: The name for the new user + :param password: The password for the new account :param admin: If true, the user will be granted full access to the account; otherwise, another user will have to add the user to the ACLs for containers to grant access. @@ -275,21 +272,21 @@ class AuthController(object): already exists, or storage url if successful """ begin = time() - if not all((new_account, new_user, new_password)): + if not all((account, user, password)): return False with self.get_conn() as conn: row = conn.execute( 'SELECT url FROM account WHERE account = ? AND user = ?', - (new_account, new_user)).fetchone() + (account, user)).fetchone() if row: self.logger.info( - 'ALREADY EXISTS create_account(%s, %s, _, %s) [%.02f]' % - (repr(new_account), repr(new_user), repr(admin), + 'ALREADY EXISTS create_user(%s, %s, _, %s) [%.02f]' % + (repr(account), repr(user), repr(admin), time() - begin)) return 'already exists' row = conn.execute( 'SELECT url, cfaccount FROM account WHERE account = ?', - (new_account,)).fetchone() + (account,)).fetchone() if row: url = row[0] account_hash = row[1] @@ -297,20 +294,20 @@ class AuthController(object): account_hash = self.add_storage_account() if not account_hash: self.logger.info( - 'FAILED create_account(%s, %s, _, %s) [%.02f]' % - (repr(new_account), repr(new_user), repr(admin), + 'FAILED create_user(%s, %s, _, %s) [%.02f]' % + (repr(account), repr(user), repr(admin), time() - begin)) return False url = self.default_cluster_url.rstrip('/') + '/' + account_hash conn.execute('''INSERT INTO account (account, url, cfaccount, user, password, admin) VALUES (?, ?, ?, ?, ?, ?)''', - (new_account, url, account_hash, new_user, new_password, + (account, url, account_hash, user, password, admin and 't' or '')) conn.commit() self.logger.info( - 'SUCCESS create_account(%s, %s, _, %s) = %s [%.02f]' % - (repr(new_account), repr(new_user), repr(admin), repr(url), + 'SUCCESS create_user(%s, %s, _, %s) = %s [%.02f]' % + (repr(account), repr(user), repr(admin), repr(url), time() - begin)) return url @@ -342,8 +339,10 @@ class AuthController(object): Valid URL paths: * GET /token/ - If the HTTP equest returns with a 204, then the token is valid, - and the TTL of the token will be available in the X-Auth-Ttl header. + If the HTTP request returns with a 204, then the token is valid, the + TTL of the token will be available in the X-Auth-Ttl header, and a + comma separated list of the "groups" the user belongs to will be in the + X-Auth-Groups header. :param request: webob.Request object """ @@ -359,7 +358,7 @@ class AuthController(object): if validation[3]: # admin access to a cfaccount groups.append(validation[3]) return HTTPNoContent(headers={'X-Auth-TTL': validation[0], - 'X-Auth-User': ','.join(groups)}) + 'X-Auth-Groups': ','.join(groups)}) def handle_add_user(self, request): """ @@ -387,7 +386,7 @@ class AuthController(object): if 'X-Auth-User-Key' not in request.headers: return HTTPBadRequest('X-Auth-User-Key is required') password = request.headers['x-auth-user-key'] - storage_url = self.create_account(account_name, user_name, password, + storage_url = self.create_user(account_name, user_name, password, request.headers.get('x-auth-user-admin') == 'true') if storage_url == 'already exists': return HTTPBadRequest(storage_url) diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index e433c7fda0..421770a241 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -39,19 +39,20 @@ class DevAuth(object): """ Accepts a standard WSGI application call, authenticating the request and installing callback hooks for authorization and ACL header - validation. + validation. For an authenticated request, REMOTE_USER will be set to a + comma separated list of the user's groups. """ - user = None + groups = None token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) if token: memcache_client = cache_from_env(env) key = 'devauth/%s' % token cached_auth_data = memcache_client.get(key) if cached_auth_data: - start, expiration, user = cached_auth_data + start, expiration, groups = cached_auth_data if time() - start > expiration: - user = None - if not user: + groups = None + if not groups: with Timeout(self.timeout): conn = http_connect(self.auth_host, self.auth_port, 'GET', '/token/%s' % token, ssl=self.ssl) @@ -61,10 +62,10 @@ class DevAuth(object): if resp.status // 100 != 2: return HTTPUnauthorized()(env, start_response) expiration = float(resp.getheader('x-auth-ttl')) - user = resp.getheader('x-auth-user') - memcache_client.set(key, (time(), expiration, user), + groups = resp.getheader('x-auth-groups') + memcache_client.set(key, (time(), expiration, groups), timeout=expiration) - env['REMOTE_USER'] = user + env['REMOTE_USER'] = groups env['swift.authorize'] = self.authorize env['swift.clean_acl'] = clean_acl return self.app(env, start_response) diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index cdaffc33e5..da737c0da8 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -106,7 +106,7 @@ class TestAuthServer(unittest.TestCase): def test_validate_token_non_existant_token(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -117,7 +117,7 @@ class TestAuthServer(unittest.TestCase): def test_validate_token_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -132,7 +132,7 @@ class TestAuthServer(unittest.TestCase): try: auth_server.time = lambda: 1 auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account('test', 'tester', + cfaccount = self.controller.create_user('test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -146,24 +146,24 @@ class TestAuthServer(unittest.TestCase): finally: auth_server.time = orig_time - def test_create_account_no_new_account(self): + def test_create_user_no_new_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('', 'tester', 'testing') + result = self.controller.create_user('', 'tester', 'testing') self.assertFalse(result) - def test_create_account_no_new_user(self): + def test_create_user_no_new_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('test', '', 'testing') + result = self.controller.create_user('test', '', 'testing') self.assertFalse(result) - def test_create_account_no_new_password(self): + def test_create_user_no_new_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - result = self.controller.create_account('test', 'tester', '') + result = self.controller.create_user('test', 'tester', '') self.assertFalse(result) - def test_create_account_good(self): + def test_create_user_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') self.assert_(url) self.assertEquals('/'.join(url.split('/')[:-1]), self.controller.default_cluster_url.rstrip('/'), repr(url)) @@ -176,7 +176,7 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_one(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test', 'tester', 'testing') + self.controller.create_user('test', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) rv = self.controller.recreate_accounts() self.assertEquals(rv.split()[0], '1', repr(rv)) @@ -184,13 +184,13 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_several(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test1', 'tester', 'testing') + self.controller.create_user('test1', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test2', 'tester', 'testing') + self.controller.create_user('test2', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test3', 'tester', 'testing') + self.controller.create_user('test3', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201) - self.controller.create_account('test4', 'tester', 'testing') + self.controller.create_user('test4', 'tester', 'testing') auth_server.http_connect = fake_http_connect(201, 201, 201, 201, 201, 201, 201, 201, 201, @@ -201,7 +201,7 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_one_fail(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') cfaccount = url.split('/')[-1] auth_server.http_connect = fake_http_connect(500, 500, 500) rv = self.controller.recreate_accounts() @@ -211,16 +211,16 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_several_fail(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test1', 'tester', 'testing') + url = self.controller.create_user('test1', 'tester', 'testing') cfaccounts = [url.split('/')[-1]] auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test2', 'tester', 'testing') + url = self.controller.create_user('test2', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test3', 'tester', 'testing') + url = self.controller.create_user('test3', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test4', 'tester', 'testing') + url = self.controller.create_user('test4', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(500, 500, 500, 500, 500, 500, @@ -233,16 +233,16 @@ class TestAuthServer(unittest.TestCase): def test_recreate_accounts_several_fail_some(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test1', 'tester', 'testing') + url = self.controller.create_user('test1', 'tester', 'testing') cfaccounts = [url.split('/')[-1]] auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test2', 'tester', 'testing') + url = self.controller.create_user('test2', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test3', 'tester', 'testing') + url = self.controller.create_user('test3', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test4', 'tester', 'testing') + url = self.controller.create_user('test4', 'tester', 'testing') cfaccounts.append(url.split('/')[-1]) auth_server.http_connect = fake_http_connect(500, 500, 500, 201, 201, 201, @@ -263,7 +263,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_missing_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -279,7 +279,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/testbad/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -294,7 +294,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -309,7 +309,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -324,7 +324,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -336,7 +336,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_good_Mosso_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -348,7 +348,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_SOSO_bad_Mosso_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing',).split('/')[-1] res = self.controller.handle_auth(Request.blank('/v1/test/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -368,7 +368,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_missing_headers(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'})) @@ -384,7 +384,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_header_format(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -399,7 +399,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_account(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -414,7 +414,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_user(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -429,7 +429,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_bad_password(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -444,7 +444,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_good(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -456,7 +456,7 @@ class TestAuthServer(unittest.TestCase): def test_auth_Mosso_good_SOSO_header_names(self): auth_server.http_connect = fake_http_connect(201, 201, 201) - cfaccount = self.controller.create_account( + cfaccount = self.controller.create_user( 'test', 'tester', 'testing').split('/')[-1] res = self.controller.handle_auth(Request.blank('/auth', environ={'REQUEST_METHOD': 'GET'}, @@ -473,9 +473,9 @@ class TestAuthServer(unittest.TestCase): logger.logger.addHandler(log_handler) try: auth_server.http_connect = fake_http_connect(201, 201, 201) - url = self.controller.create_account('test', 'tester', 'testing') + url = self.controller.create_user('test', 'tester', 'testing') self.assertEquals(log.getvalue().rsplit(' ', 1)[0], - "auth SUCCESS create_account('test', 'tester', _, False) = %s" + "auth SUCCESS create_user('test', 'tester', _, False) = %s" % repr(url)) log.truncate(0) def start_response(*args): From 2edfd2b951681641d13fceb98179140bc332b141 Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Sep 2010 20:30:09 -0700 Subject: [PATCH 09/16] Doc fixes and updates --- doc/source/auth.rst | 2 +- doc/source/overview_auth.rst | 27 ++++++++++++++++----------- swift/auth/server.py | 14 +++++++++----- swift/common/client.py | 4 ++-- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/doc/source/auth.rst b/doc/source/auth.rst index dc5a65ac45..feb3be8a99 100644 --- a/doc/source/auth.rst +++ b/doc/source/auth.rst @@ -4,7 +4,7 @@ Developer's Authorization ************************* -.. _auth-server: +.. _auth_server: Auth Server =========== diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst index a242280553..364a6928dc 100644 --- a/doc/source/overview_auth.rst +++ b/doc/source/overview_auth.rst @@ -6,9 +6,9 @@ The Auth System Developer Auth -------------- -The auth system for Swift is based on the auth system from the existing -Rackspace architecture -- actually from a few existing auth systems -- -and is therefore a bit disjointed. The distilled points about it are: +The auth system for Swift is loosely based on the auth system from the existing +Rackspace architecture -- actually from a few existing auth systems -- and is +therefore a bit disjointed. The distilled points about it are: * The authentication/authorization part is outside Swift itself * The user of Swift passes in an auth token with each request @@ -23,13 +23,16 @@ of something unique, some use "something else" but the salient point is that the token is a string which can be sent as-is back to the auth system for validation. -An auth call is given the auth token and the Swift account hash. For a valid -token, the auth system responds with a session TTL and overall expiration in -seconds from now. Swift does not honor the session TTL but will cache the -token up to the expiration time. Tokens can be purged through a call to the -auth system. +Swift will make calls to the external auth system, giving the auth token to be +validated. For a valid token, the auth system responds with an overall +expiration in seconds from now. Swift will cache the token up to the expiration +time. The included devauth also has the concept of admin and non-admin users +within an account. Admin users can do anything within the account. Non-admin +users can only perform operations per container based on the container's +X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see +:mod:`swift.common.middleware.acl` -The user starts a session by sending a ReST request to that auth system +The user starts a session by sending a ReST request to the external auth system to receive the auth token and a URL to the Swift system. -------------- @@ -40,8 +43,10 @@ Auth is written as wsgi middleware, so implementing your own auth is as easy as writing new wsgi middleware, and plugging it in to the proxy server. The current middleware is implemented in the DevAuthMiddleware class in -swift/common/auth.py, and should be a good starting place for implemeting -your own auth. +swift/common/middleware/auth.py, and should be a good starting place for +implementing your own auth. + +Also, see :doc:`development_auth`. ------------------ History and Future diff --git a/swift/auth/server.py b/swift/auth/server.py index e047397072..d9007e1e62 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -65,11 +65,12 @@ class AuthController(object): the token. * The Swift cluster completes the user's request. - Another use case is creating a new account: + Another use case is creating a new user: - * The developer makes a ReST call to create a new account. - * The auth server makes ReST calls to the Swift cluster's account servers - to create a new account on its end. + * The developer makes a ReST call to create a new user. + * If the account for the user does not yet exist, the auth server makes + ReST calls to the Swift cluster's account servers to create a new account + on its end. * The auth server records the information in its database. A last use case is recreating existing accounts; this is really only useful @@ -261,6 +262,9 @@ class AuthController(object): cluster while still supporting old accounts going to the Swift clusters they were created on. + Currently, updating a user's information (password, admin access) must + be done by directly updating the sqlite database. + :param account: The name for the new account :param user: The name for the new user :param password: The password for the new account @@ -334,7 +338,7 @@ class AuthController(object): def handle_token(self, request): """ - Hanles ReST request from Swift to validate tokens + Handles ReST requests from Swift to validate tokens Valid URL paths: * GET /token/ diff --git a/swift/common/client.py b/swift/common/client.py index 4832452130..0448442526 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -626,12 +626,12 @@ def put_object(url, token, container, name, contents, content_length=None, def post_object(url, token, container, name, headers, http_conn=None): """ - Change object metadata + Update object metadata :param url: storage URL :param token: auth token :param container: container name that the object is in - :param name: object name to change + :param name: name of the object to update :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) From d0367fdf1909ad0613e094473af23b61f8b6d34f Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Sep 2010 21:06:16 -0700 Subject: [PATCH 10/16] Updated direct_client to match the changes in client --- bin/swift-stats-report | 2 +- swift/account/reaper.py | 2 +- swift/common/client.py | 8 --- swift/common/direct_client.py | 71 +++++++++++++++----------- test/probe/test_object_async_update.py | 4 +- test/probe/test_object_handoff.py | 22 ++++---- test/unit/common/test_direct_client.py | 1 - 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/bin/swift-stats-report b/bin/swift-stats-report index bbb4410d2b..3f735877cf 100755 --- a/bin/swift-stats-report +++ b/bin/swift-stats-report @@ -86,7 +86,7 @@ def audit(coropool, connpool, account, container_ring, object_ring, options): retries_done[0] += attempts - 1 found = True if not estimated_objects: - estimated_objects = info[0] + estimated_objects = int(info['x-container-object-count']) except ClientException, err: if err.http_status not in (404, 507): error_log('Giving up on /%s/%s/%s: %s' % (part, account, diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 02f1ab76ad..87fa026ebd 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -310,7 +310,7 @@ class AccountReaper(Daemon): try: objects = direct_get_container(node, part, account, container, marker=marker, conn_timeout=self.conn_timeout, - response_timeout=self.node_timeout) + response_timeout=self.node_timeout)[1] self.stats_return_codes[2] = \ self.stats_return_codes.get(2, 0) + 1 except ClientException, err: diff --git a/swift/common/client.py b/swift/common/client.py index 0448442526..e17afee958 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -504,10 +504,6 @@ def get_object(url, token, container, name, http_conn=None, raise ClientException('Object GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) if resp_chunk_size: def _object_body(): @@ -550,10 +546,6 @@ def head_object(url, token, container, name, http_conn=None): raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py index 1d7030c09d..71fe3005a4 100644 --- a/swift/common/direct_client.py +++ b/swift/common/direct_client.py @@ -47,7 +47,8 @@ def direct_head_container(node, part, account, container, conn_timeout=5, :param container: container name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (object count, bytes used) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s' % (account, container) with Timeout(conn_timeout): @@ -65,8 +66,10 @@ def direct_head_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return int(resp.getheader('x-container-object-count')), \ - int(resp.getheader('x-container-bytes-used')) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_container(node, part, account, container, marker=None, @@ -85,7 +88,8 @@ def direct_get_container(node, part, account, container, marker=None, :param delimeter: delimeter for the query :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: list of objects + :returns: a tuple of (response headers, a list of objects) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s' % (account, container) qs = 'format=json' @@ -111,10 +115,13 @@ def direct_get_container(node, part, account, container, marker=None, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value if resp.status == 204: resp.read() - return [] - return json_loads(resp.read()) + return resp_headers, [] + return resp_headers, json_loads(resp.read()) def direct_delete_container(node, part, account, container, conn_timeout=5, @@ -126,6 +133,7 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Container server %s:%s direct DELETE %s gave status %s' % @@ -135,7 +143,6 @@ def direct_delete_container(node, part, account, container, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def direct_head_object(node, part, account, container, obj, conn_timeout=5, @@ -150,8 +157,8 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: tuple of (content-type, object size, last modified timestamp, - etag, metadata dictionary) + :returns: a dict containing the response's headers (all header names will + be lowercase) """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -169,19 +176,14 @@ def direct_head_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return resp.getheader('content-type'), \ - int(resp.getheader('content-length')), \ - resp.getheader('last-modified'), \ - resp.getheader('etag').strip('"'), \ - metadata + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers def direct_get_object(node, part, account, container, obj, conn_timeout=5, - response_timeout=15): + response_timeout=15, resp_chunk_size=None): """ Get object directly from the object server. @@ -192,7 +194,9 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, :param obj: object name :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response - :returns: object + :param resp_chunk_size: if defined, chunk size of data to read. + :returns: a tuple of (response headers, the object's contents) The response + headers will be a dict and all header names will be lowercase. """ path = '/%s/%s/%s' % (account, container, obj) with Timeout(conn_timeout): @@ -201,6 +205,7 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, with Timeout(response_timeout): resp = conn.getresponse() if resp.status < 200 or resp.status >= 300: + resp.read() raise ClientException( 'Object server %s:%s direct GET %s gave status %s' % (node['ip'], node['port'], @@ -209,16 +214,20 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) - return (resp.getheader('content-type'), - int(resp.getheader('content-length')), - resp.getheader('last-modified'), - resp.getheader('etag').strip('"'), - metadata, - resp.read()) + if resp_chunk_size: + + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, object_body def direct_delete_object(node, part, account, container, obj, @@ -242,6 +251,7 @@ def direct_delete_object(node, part, account, container, obj, 'DELETE', path, headers) with Timeout(response_timeout): resp = conn.getresponse() + resp.read() if resp.status < 200 or resp.status >= 300: raise ClientException( 'Object server %s:%s direct DELETE %s gave status %s' % @@ -251,7 +261,6 @@ def direct_delete_object(node, part, account, container, obj, http_host=node['ip'], http_port=node['port'], http_device=node['device'], http_status=resp.status, http_reason=resp.reason) - return resp def retry(func, *args, **kwargs): diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index 4b4c3a49b7..a5d5852c68 100755 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -52,7 +52,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container(cnode, cpart, - self.account, container)) + self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-object-updater', @@ -60,7 +60,7 @@ class TestObjectAsyncUpdate(unittest.TestCase): for p in ps: p.wait() objs = [o['name'] for o in direct_client.direct_get_container(cnode, - cpart, self.account, container)] + cpart, self.account, container)[1]] self.assert_(obj in objs) diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 37767cbb5e..006f0d3a1e 100755 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -81,7 +81,7 @@ class TestObjectHandoff(unittest.TestCase): for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, - self.account, container)] + self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % @@ -127,9 +127,9 @@ class TestObjectHandoff(unittest.TestCase): kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.post_object(self.url, self.token, container, obj, headers={'x-object-meta-probe': 'value'}) - ometadata = client.head_object(self.url, self.token, container, obj) - if ometadata.get('x-object-meta-probe') != 'value': - raise Exception('Metadata incorrect, was %s' % repr(ometadata)) + oheaders = client.head_object(self.url, self.token, container, obj) + if oheaders.get('x-object-meta-probe') != 'value': + raise Exception('Metadata incorrect, was %s' % repr(oheaders)) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, @@ -144,9 +144,9 @@ class TestObjectHandoff(unittest.TestCase): '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') == 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') == 'value': raise Exception('Previously downed object server had the new ' 'metadata when it should not have it') # Run the extra server last so it'll remove it's extra partition @@ -160,9 +160,9 @@ class TestObjectHandoff(unittest.TestCase): call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) - ometadata = direct_client.direct_get_object(onode, opart, self.account, - container, obj)[-2] - if ometadata.get('probe') != 'value': + oheaders = direct_client.direct_get_object(onode, opart, self.account, + container, obj)[0] + if oheaders.get('x-object-meta-probe') != 'value': raise Exception( 'Previously downed object server did not have the new metadata') @@ -182,7 +182,7 @@ class TestObjectHandoff(unittest.TestCase): for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( - cnode, cpart, self.account, container)] + cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py index 029791289e..a925c118bb 100644 --- a/test/unit/common/test_direct_client.py +++ b/test/unit/common/test_direct_client.py @@ -16,7 +16,6 @@ # TODO: Tests import unittest -from swift.common import direct_client class TestAuditor(unittest.TestCase): From 85b8d970863ea78cc8fe90fe30e1e993e64ebac3 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 6 Sep 2010 13:26:31 -0700 Subject: [PATCH 11/16] Referrers now support user:pass part of URLs; rest of tests for what has changed --- swift/common/middleware/acl.py | 5 +- test/unit/auth/test_server.py | 55 ++++++- test/unit/common/middleware/test_acl.py | 129 ++++++---------- test/unit/common/middleware/test_auth.py | 183 ++++++++++++++++++++++- test/unit/proxy/test_server.py | 183 +++++++++++++++++++++++ 5 files changed, 466 insertions(+), 89 deletions(-) diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index bf3148a16a..15e1f05989 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -139,7 +139,10 @@ def referrer_allowed(referrer, referrer_acl): else: parts = referrer.split('//', 1) if len(parts) == 2: - rhost = parts[1].split('/', 1)[0].split(':', 1)[0].lower() + rhost = parts[1].split('/', 1)[0] + if '@' in rhost: + rhost = rhost.rsplit('@', 1)[1] + rhost = rhost.split(':', 1)[0].lower() else: rhost = 'unknown' for mhost in referrer_acl: diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py index da737c0da8..791fc3853f 100644 --- a/test/unit/auth/test_server.py +++ b/test/unit/auth/test_server.py @@ -21,10 +21,11 @@ from StringIO import StringIO from uuid import uuid4 from logging import StreamHandler +import sqlite3 from webob import Request from swift.auth import server as auth_server -from swift.common.db import DatabaseConnectionError +from swift.common.db import DatabaseConnectionError, get_db_connection from swift.common.utils import get_logger @@ -576,6 +577,58 @@ class TestAuthServer(unittest.TestCase): auth_server.Request = orig_Request logger.logger.handlers.remove(log_handler) + def test_upgrading_from_db1(self): + swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex + os.mkdir(swift_dir) + try: + # Create db1 + db_file = os.path.join(swift_dir, 'auth.db') + conn = get_db_connection(db_file, okay_to_create=True) + conn.execute('''CREATE TABLE IF NOT EXISTS account ( + account TEXT, url TEXT, cfaccount TEXT, + user TEXT, password TEXT)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account + ON account (account)''') + conn.execute('''CREATE TABLE IF NOT EXISTS token ( + cfaccount TEXT, token TEXT, created FLOAT)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount + ON token (cfaccount)''') + conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created + ON token (created)''') + conn.execute('''INSERT INTO account + (account, url, cfaccount, user, password) + VALUES ('act', 'url', 'cfa', 'usr', 'pas')''') + conn.execute('''INSERT INTO token (cfaccount, token, created) + VALUES ('cfa', 'tok', '1')''') + conn.commit() + conn.close() + # Upgrade to current db + conf = {'swift_dir': swift_dir} + controller = auth_server.AuthController(conf, FakeRing()) + # Check new items exist and are correct + conn = get_db_connection(db_file) + row = conn.execute('SELECT admin FROM account').fetchone() + self.assertEquals(row[0], 't') + row = conn.execute('SELECT user FROM token').fetchone() + self.assert_(not row) + finally: + rmtree(swift_dir) + + def test_create_user_twice(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.controller.create_user('test', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + self.assertEquals( + self.controller.create_user('test', 'tester', 'testing'), + 'already exists') + + def test_create_2users_1account(self): + auth_server.http_connect = fake_http_connect(201, 201, 201) + url = self.controller.create_user('test', 'tester', 'testing') + auth_server.http_connect = fake_http_connect(201, 201, 201) + url2 = self.controller.create_user('test', 'tester2', 'testing2') + self.assertEquals(url, url2) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_acl.py b/test/unit/common/middleware/test_acl.py index 69de606849..26838f78df 100644 --- a/test/unit/common/middleware/test_acl.py +++ b/test/unit/common/middleware/test_acl.py @@ -13,95 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import with_statement -import logging -import os -import sys import unittest -from contextlib import contextmanager - -import eventlet -from webob import Request from swift.common.middleware import acl -# mocks -logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - -class FakeMemcache(object): - def __init__(self): - self.store = {} - - def get(self, key): - return self.store.get(key) - - def set(self, key, value, timeout=0): - self.store[key] = value - return True - - def incr(self, key, timeout=0): - self.store[key] = self.store.setdefault(key, 0) + 1 - return self.store[key] - - @contextmanager - def soft_lock(self, key, timeout=0, retries=5): - yield True - - def delete(self, key): - try: - del self.store[key] - except: - pass - return True - - -def mock_http_connect(response, headers=None, with_exc=False): - class FakeConn(object): - def __init__(self, status, headers, with_exc): - self.status = status - self.reason = 'Fake' - self.host = '1.2.3.4' - self.port = '1234' - self.with_exc = with_exc - self.headers = headers - if self.headers is None: - self.headers = {} - def getresponse(self): - if self.with_exc: - raise Exception('test') - return self - def getheader(self, header): - return self.headers[header] - def read(self, amt=None): - return '' - def close(self): - return - return lambda *args, **kwargs: FakeConn(response, headers, with_exc) - - -class Logger(object): - def __init__(self): - self.error_value = None - self.exception_value = None - def error(self, msg, *args, **kwargs): - self.error_value = (msg, args, kwargs) - def exception(self, msg, *args, **kwargs): - _, exc, _ = sys.exc_info() - self.exception_value = (msg, - '%s %s' % (exc.__class__.__name__, str(exc)), args, kwargs) -# tests - -class FakeApp(object): - def __call__(self, env, start_response): - return "OK" - -def start_response(*args): - pass - -class TestAuth(unittest.TestCase): - # I brought these over from another refactor I've been trying, but they - # need work. +class TestACL(unittest.TestCase): def test_clean_acl(self): value = acl.clean_acl('header', '.ref:any') @@ -118,6 +35,9 @@ class TestAuth(unittest.TestCase): self.assertEquals(value, '.ref:any,.ref:-.ending.with') value = acl.clean_acl('header', '.ref:one,.ref:-two') self.assertEquals(value, '.ref:one,.ref:-two') + value = acl.clean_acl('header', + '.ref:one,.ref:-two,account,account:user') + self.assertEquals(value, '.ref:one,.ref:-two,account,account:user') value = acl.clean_acl('header', ' .ref : one , ,, .ref:two , .ref : - three ') self.assertEquals(value, '.ref:one,.ref:two,.ref:-three') @@ -129,6 +49,7 @@ class TestAuth(unittest.TestCase): self.assertRaises(ValueError, acl.clean_acl, 'header', ' .ref : - ') self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .ref : - ') + self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.ref:r') def test_parse_acl(self): self.assertEquals(acl.parse_acl(None), ([], [])) @@ -145,6 +66,46 @@ class TestAuth(unittest.TestCase): (['ref3', 'ref5', '-ref6'], ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4'])) + def test_referrer_allowed(self): + self.assert_(not acl.referrer_allowed('host', None)) + self.assert_(not acl.referrer_allowed('host', [])) + self.assert_(acl.referrer_allowed(None, ['any'])) + self.assert_(acl.referrer_allowed('', ['any'])) + self.assert_(not acl.referrer_allowed(None, ['specific.host'])) + self.assert_(not acl.referrer_allowed('', ['specific.host'])) + self.assert_(acl.referrer_allowed('http://www.example.com/index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user@www.example.com/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://www.example.com:8080/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user@www.example.com:8080/index.html', ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com:8080/index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed( + 'http://user:pass@www.example.com:8080', ['.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['.example.com'])) + self.assert_(not acl.referrer_allowed('http://thief.example.com', + ['.example.com', '-thief.example.com'])) + self.assert_(not acl.referrer_allowed('http://thief.example.com', + ['any', '-thief.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['.other.com', 'www.example.com'])) + self.assert_(acl.referrer_allowed('http://www.example.com', + ['-.example.com', 'www.example.com'])) + # This is considered a relative uri to the request uri, a mode not + # currently supported. + self.assert_(not acl.referrer_allowed('www.example.com', + ['.example.com'])) + self.assert_(not acl.referrer_allowed('../index.html', + ['.example.com'])) + self.assert_(acl.referrer_allowed('www.example.com', ['any'])) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index d48ce39202..60451ff5d2 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -94,14 +94,191 @@ class Logger(object): class FakeApp(object): def __call__(self, env, start_response): - return "OK" + return ['204 No Content'] def start_response(*args): pass class TestAuth(unittest.TestCase): - # TODO: With the auth refactor, these tests have to be refactored as well. - pass + + def setUp(self): + self.test_auth = auth.filter_factory({})(FakeApp()) + + def test_auth_fail(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(404) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': FakeMemcache()}, + lambda x, y: None)) + self.assert_(result.startswith('401'), result) + finally: + auth.http_connect = old_http_connect + + def test_auth_success(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': FakeMemcache()}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + finally: + auth.http_connect = old_http_connect + + def test_auth_memcache(self): + old_http_connect = auth.http_connect + try: + fake_memcache = FakeMemcache() + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + auth.http_connect = mock_http_connect(404) + # Should still be in memcache + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + finally: + auth.http_connect = old_http_connect + + def test_auth_just_expired(self): + old_http_connect = auth.http_connect + try: + fake_memcache = FakeMemcache() + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,cfa'}) + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('204'), result) + auth.http_connect = mock_http_connect(404) + # Should still be in memcache, but expired + result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', + 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + lambda x, y: None)) + self.assert_(result.startswith('401'), result) + finally: + auth.http_connect = old_http_connect + + def test_middleware_success(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + req = Request.blank('/v/a/c/o', headers={'x-auth-token': 't'}) + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assertEquals(req.remote_user, 'act:usr,act,cfa') + finally: + auth.http_connect = old_http_connect + + def test_middleware_no_header(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + req = Request.blank('/v/a/c/o') + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assert_(not req.remote_user, req.remote_user) + finally: + auth.http_connect = old_http_connect + + def test_middleware_storage_token(self): + old_http_connect = auth.http_connect + try: + auth.http_connect = mock_http_connect(204, + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + req = Request.blank('/v/a/c/o', headers={'x-storage-token': 't'}) + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, start_response)) + self.assert_(result.startswith('204'), result) + self.assertEquals(req.remote_user, 'act:usr,act,cfa') + finally: + auth.http_connect = old_http_connect + + def test_authorize_bad_path(self): + req = Request.blank('/badpath') + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/badpath') + req.remote_user = 'act:usr,act,cfa' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_account_access(self): + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act,cfa' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_acl_group_access(self): + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act:usr' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act2' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = 'act:usr2' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + + def test_authorize_acl_referrer_access(self): + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = '.ref:any' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.acl = '.ref:.example.com' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + req = Request.blank('/v1/cfa') + req.remote_user = 'act:usr,act' + req.referrer = 'http://www.example.com/index.html' + req.acl = '.ref:.example.com' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/v1/cfa') + req.acl = '.ref:any' + self.assertEquals(self.test_auth.authorize(req), None) + req = Request.blank('/v1/cfa') + req.acl = '.ref:.example.com' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('401'), resp) + req = Request.blank('/v1/cfa') + req.referrer = 'http://www.example.com/index.html' + req.acl = '.ref:.example.com' + self.assertEquals(self.test_auth.authorize(req), None) if __name__ == '__main__': diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0db4e6df47..941a3e90a1 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -34,6 +34,7 @@ from eventlet import sleep, spawn, TimeoutError, util, wsgi, listen from eventlet.timeout import Timeout import simplejson from webob import Request +from webob.exc import HTTPUnauthorized from test.unit import connect_tcp, readuntil2crlfs from swift.proxy import server as proxy_server @@ -208,6 +209,35 @@ class TestProxyServer(unittest.TestCase): resp = app.handle_request(req) self.assertEquals(resp.status_int, 500) + def test_calls_authorize_allow(self): + called = [False] + def authorize(req): + called[0] = True + with save_globals(): + proxy_server.http_connect = fake_http_connect(200) + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), container_ring=FakeRing(), + object_ring=FakeRing()) + req = Request.blank('/v1/a') + req.environ['swift.authorize'] = authorize + app.update_request(req) + resp = app.handle_request(req) + self.assert_(called[0]) + + def test_calls_authorize_deny(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + app = proxy_server.Application(None, FakeMemcache(), + account_ring=FakeRing(), container_ring=FakeRing(), + object_ring=FakeRing()) + req = Request.blank('/v1/a') + req.environ['swift.authorize'] = authorize + app.update_request(req) + resp = app.handle_request(req) + self.assert_(called[0]) + class TestObjectController(unittest.TestCase): @@ -1510,6 +1540,73 @@ class TestObjectController(unittest.TestCase): finally: self.app.object_chunk_size = orig_object_chunk_size + def test_GET_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.GET(req) + self.assert_(called[0]) + + def test_HEAD_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', {'REQUEST_METHOD': 'HEAD'}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.HEAD(req) + self.assert_(called[0]) + + def test_POST_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'POST'}, + headers={'Content-Length': '5'}, body='12345') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + + def test_PUT_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 200, 201, 201, 201) + controller = proxy_server.ObjectController(self.app, 'account', + 'container', 'object') + req = Request.blank('/a/c/o', environ={'REQUEST_METHOD': 'PUT'}, + headers={'Content-Length': '5'}, body='12345') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + + class TestContainerController(unittest.TestCase): "Test swift.proxy_server.ContainerController" @@ -1855,6 +1952,92 @@ class TestContainerController(unittest.TestCase): resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) + def test_POST_calls_clean_acl(self): + called = [False] + def clean_acl(header, value): + called[0] = True + raise ValueError('fake error') + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Container-Read': '.ref:any'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + called[0] = False + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, + headers={'X-Container-Write': '.ref:any'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.POST(req) + self.assert_(called[0]) + + def test_PUT_calls_clean_acl(self): + called = [False] + def clean_acl(header, value): + called[0] = True + raise ValueError('fake error') + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Container-Read': '.ref:any'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + called[0] = False + with save_globals(): + proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, + headers={'X-Container-Write': '.ref:any'}) + req.environ['swift.clean_acl'] = clean_acl + self.app.update_request(req) + res = controller.PUT(req) + self.assert_(called[0]) + + def test_GET_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c') + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.GET(req) + self.assert_(called[0]) + + def test_HEAD_calls_authorize(self): + called = [False] + def authorize(req): + called[0] = True + return HTTPUnauthorized(request=req) + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(200, 201, 201, 201) + controller = proxy_server.ContainerController(self.app, 'account', + 'container') + req = Request.blank('/a/c', {'REQUEST_METHOD': 'HEAD'}) + req.environ['swift.authorize'] = authorize + self.app.update_request(req) + res = controller.HEAD(req) + self.assert_(called[0]) + class TestAccountController(unittest.TestCase): From d2ec027e2261e02b76b5da1712d099e35f55a90d Mon Sep 17 00:00:00 2001 From: gholt Date: Wed, 8 Sep 2010 22:37:27 -0700 Subject: [PATCH 12/16] For ACL strings: Shortened .ref to just .r, though .ref, .referer, and .referrer are all accepted. Updated 'Creating Your Own Auth Middleware' to describe how the DevAuth server works and suggestions for creating one's own. Added reseller_prefix (optional) implementation. Used urlparse in referrer_allowed. Fixed bug where group names would get lowercased by clean_acl. Changed .r:any to .r:*. Allowed .r:*.example.com to mean .r:.example.com. Made proxy log just the first authenticated group (the user) alongside the token. Moved proxy callback to clean_acl before the length check of the metadata. Cleaned up redundant logic in first proxy swift.authorize callback. Bit better docs. More and updated tests. --- bin/st | 5 +- doc/source/development_auth.rst | 45 ++++++++-- etc/proxy-server.conf-sample | 4 + swift/auth/server.py | 11 ++- swift/common/middleware/acl.py | 95 +++++++++++----------- swift/common/middleware/auth.py | 9 +- swift/proxy/server.py | 19 +++-- test/functional/sample.conf | 1 + test/functionalnosetests/swift_testing.py | 6 ++ test/functionalnosetests/test_container.py | 27 +++--- test/functionalnosetests/test_object.py | 2 +- test/unit/common/middleware/test_acl.py | 85 +++++++++++-------- test/unit/common/middleware/test_auth.py | 16 ++-- test/unit/container/test_server.py | 10 +-- test/unit/proxy/test_server.py | 8 +- 15 files changed, 209 insertions(+), 134 deletions(-) diff --git a/bin/st b/bin/st index f7546b7472..7d672b1eb9 100755 --- a/bin/st +++ b/bin/st @@ -1417,9 +1417,8 @@ Example: 'general documentation for what this means).') parser.add_option('-r', '--read-acl', dest='read_acl', help='Sets the Read ACL with post container commands. ' - 'Quick summary of ACL syntax: .ref:any, ' - '.ref:-.example.com, .ref:www.example.com, account1, ' - 'account2:user2') + 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' + '.r:www.example.com, account1, account2:user2') parser.add_option('-w', '--write-acl', dest='write_acl', help='Sets the Write ACL with post container commands. ' 'Quick summary of ACL syntax: account1, account2:user2') diff --git a/doc/source/development_auth.rst b/doc/source/development_auth.rst index fce52187b5..51741dcca5 100644 --- a/doc/source/development_auth.rst +++ b/doc/source/development_auth.rst @@ -1,10 +1,10 @@ -=============== -Auth Middleware -=============== +========================== +Auth Server and Middleware +========================== ---------------------------------- -Creating Your Own Auth Middleware ---------------------------------- +-------------------------------------------- +Creating Your Own Auth Server and Middleware +-------------------------------------------- The included swift/common/middleware/auth.py is a good minimal example of how to create auth middleware. The main points are that the auth middleware can @@ -26,6 +26,39 @@ specific information, it just passes it along. Convention has environ['REMOTE_USER'] set to the authenticated user string but often more information is needed than just that. +The included DevAuth will set the REMOTE_USER to a comma separated list of +groups the user belongs to. The first group will be the "user's group", a group +that only the user belongs to. The second group will be the "account's group", +a group that includes all users for that auth account (different than the +storage account). The third group is optional and is the storage account +string. If the user does not have admin access to the account, the third group +will be omitted. + +It is highly recommended that authentication server implementers prefix their +group names and tokens with a configurable reseller prefix (`AUTH_` by default +with the included DevAuth). This prefix will allow deconflicting with other +authentication servers that might be using the same Swift cluster. + +The only other restriction is that no group name should begin with a period '.' +as that is reserved for internal Swift use (such as the .r for referrer +designations as you'll see later). This shouldn't be an issue if a reseller +prefix is in use and does not begin with a period. + +Example Authentication with DevAuth: + + * Token AUTH_tkabcd is given to the DevAuth middleware in a request's + X-Auth-Token header. + * The DevAuth middleware makes a validate token AUTH_tkabcd call to the + external DevAuth server. + * The external DevAuth server validates the token AUTH_tkabcd and discovers + it matches the "tester" user within the "test" account for the storage + account "AUTH_storage_xyz". + * The external DevAuth server responds with "X-Auth-Groups: + AUTH_test:tester,AUTH_test,AUTH_storage_xyz" + * Now this user will have full access (via authorization procedures later) + to the AUTH_storage_xyz Swift storage account and access to anything with + ACLs specifying at least one of those three groups returned. + Authorization is performed through callbacks by the Swift Proxy server to the WSGI environment's swift.authorize value, if one is set. The swift.authorize value should simply be a function that takes a webob.Request as an argument and diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 4ae7596392..6ee36451ab 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -37,6 +37,10 @@ use = egg:swift#proxy [filter:auth] use = egg:swift#auth +# The reseller prefix, if set, will verify a token begins with this prefix +# before even attempting to validate it with the external reseller. Usefull if +# multiple auth systems are in use for one Swift cluster. +# reseller_prefix = # ip = 127.0.0.1 # port = 11000 # ssl = false diff --git a/swift/auth/server.py b/swift/auth/server.py index d9007e1e62..931bbb6080 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -94,6 +94,9 @@ class AuthController(object): def __init__(self, conf, ring=None): self.logger = get_logger(conf) self.swift_dir = conf.get('swift_dir', '/etc/swift') + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() + if self.reseller_prefix and self.reseller_prefix[-1] != '_': + self.reseller_prefix += '_' self.default_cluster_url = \ conf.get('default_cluster_url', 'http://127.0.0.1:8080/v1') self.token_life = int(conf.get('token_life', 86400)) @@ -147,7 +150,7 @@ class AuthController(object): begin = time() orig_account_name = account_name if not account_name: - account_name = str(uuid4()) + account_name = '%s%s' % (self.reseller_prefix, uuid4().hex) partition, nodes = self.account_ring.get_nodes(account_name) headers = {'X-Timestamp': normalize_timestamp(time()), 'x-cf-trans-id': 'tx' + str(uuid4())} @@ -358,7 +361,9 @@ class AuthController(object): validation = self.validate_token(token) if not validation: return HTTPNotFound() - groups = ['%s:%s' % (validation[1], validation[2]), validation[1]] + groups = [ + '%s%s:%s' % (self.reseller_prefix, validation[1], validation[2]), + '%s%s' % (self.reseller_prefix, validation[1])] if validation[3]: # admin access to a cfaccount groups.append(validation[3]) return HTTPNoContent(headers={'X-Auth-TTL': validation[0], @@ -482,7 +487,7 @@ class AuthController(object): if row: token = row[0] else: - token = 'tk' + str(uuid4()) + token = '%stk%s' % (self.reseller_prefix, uuid4().hex) conn.execute(''' INSERT INTO token (token, created, account, user, cfaccount) diff --git a/swift/common/middleware/acl.py b/swift/common/middleware/acl.py index 15e1f05989..6403aed726 100644 --- a/swift/common/middleware/acl.py +++ b/swift/common/middleware/acl.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from urlparse import urlparse + + def clean_acl(name, value): """ Returns a cleaned ACL header value, validating that it meets the formatting @@ -27,30 +30,35 @@ def clean_acl(name, value): The referrer designation format is:: - .ref:[-]value + .r:[-]value - The value can be "any" to specify any referrer host is allowed access, a - specific host name like "www.example.com", or if it has a leading period - "." it is a domain name specification, like ".example.com". The leading - minus sign "-" indicates referrer hosts that should be denied access. + The ``.r`` can also be ``.ref``, ``.referer``, or ``.referrer``; though it + will be shortened to just ``.r`` for decreased character count usage. + + The value can be ``*`` to specify any referrer host is allowed access, a + specific host name like ``www.example.com``, or if it has a leading period + ``.`` or leading ``*.`` it is a domain name specification, like + ``.example.com`` or ``*.example.com``. The leading minus sign ``-`` + indicates referrer hosts that should be denied access. Referrer access is applied in the order they are specified. For example, - .ref:.example.com,.ref:-thief.example.com would allow all hosts ending with + .r:.example.com,.r:-thief.example.com would allow all hosts ending with .example.com except for the specific host thief.example.com. Example valid ACLs:: - .ref:any - .ref:any,.ref:-.thief.com - .ref:any,.ref:-.thief.com,bobs_account,sues_account:sue + .r:* + .r:*,.r:-.thief.com + .r:*,.r:.example.com,.r:-thief.example.com + .r:*,.r:-.thief.com,bobs_account,sues_account:sue bobs_account,sues_account:sue Example invalid ACLs:: - .ref: - .ref:- + .r: + .r:- - Also, .ref designations aren't allowed in headers whose names include the + Also, .r designations aren't allowed in headers whose names include the word 'write'. ACLs that are "messy" will be cleaned up. Examples: @@ -58,10 +66,11 @@ def clean_acl(name, value): ====================== ====================== Original Cleaned ---------------------- ---------------------- - bob, sue bob,sue - bob , sue bob,sue - bob,,,sue bob,sue - .ref : any .ref:any + ``bob, sue`` ``bob,sue`` + ``bob , sue`` ``bob,sue`` + ``bob,,,sue`` ``bob,sue`` + ``.referrer : *`` ``.r:*`` + ``.ref:*.example.com`` ``.r:.example.com`` ====================== ====================== :param name: The name of the header being cleaned, such as X-Container-Read @@ -71,30 +80,34 @@ def clean_acl(name, value): :raises ValueError: If the value does not meet the ACL formatting requirements; the error message will indicate why. """ + name = name.lower() values = [] - for raw_value in value.lower().split(','): + for raw_value in value.split(','): raw_value = raw_value.strip() if raw_value: if ':' not in raw_value: values.append(raw_value) else: first, second = (v.strip() for v in raw_value.split(':', 1)) - if first != '.ref': + if not first or first[0] != '.': values.append(raw_value) - elif 'write' in name: - raise ValueError('Referrers not allowed in write ACLs: %s' - % repr(raw_value)) - elif not second: - raise ValueError('No value after referrer designation in ' - '%s' % repr(raw_value)) - else: - if second[0] == '-': + elif first in ('.r', '.ref', '.referer', '.referrer'): + if 'write' in name: + raise ValueError('Referrers not allowed in write ACL: ' + '%s' % repr(raw_value)) + negate = False + if second and second[0] == '-': + negate = True second = second[1:].strip() - if not second: - raise ValueError('No value after referrer deny ' - 'designation in %s' % repr(raw_value)) - second = '-' + second - values.append('%s:%s' % (first, second)) + if second and second != '*' and second[0] == '*': + second = second[1:].strip() + if not second or second == '.': + raise ValueError('No host/domain value after referrer ' + 'designation in ACL: %s' % repr(raw_value)) + values.append('.r:%s%s' % (negate and '-' or '', second)) + else: + raise ValueError('Unknown designator %s in ACL: %s' % + (repr(first), repr(raw_value))) return ','.join(values) @@ -106,15 +119,15 @@ def parse_acl(acl_string): :param acl_string: The standard Swift ACL string to parse. :returns: A tuple of (referrers, groups) where referrers is a list of - referrer designations (without the leading .ref:) and groups is a + referrer designations (without the leading .r:) and groups is a list of groups to allow access. """ referrers = [] groups = [] if acl_string: for value in acl_string.split(','): - if value.startswith('.ref:'): - referrers.append(value[len('.ref:'):]) + if value.startswith('.r:'): + referrers.append(value[len('.r:'):]) else: groups.append(value) return referrers, groups @@ -134,24 +147,14 @@ def referrer_allowed(referrer, referrer_acl): """ allow = False if referrer_acl: - if not referrer: - rhost = 'unknown' - else: - parts = referrer.split('//', 1) - if len(parts) == 2: - rhost = parts[1].split('/', 1)[0] - if '@' in rhost: - rhost = rhost.rsplit('@', 1)[1] - rhost = rhost.split(':', 1)[0].lower() - else: - rhost = 'unknown' + rhost = urlparse(referrer or '').hostname or 'unknown' for mhost in referrer_acl: if mhost[0] == '-': mhost = mhost[1:] if mhost == rhost or \ (mhost[0] == '.' and rhost.endswith(mhost)): allow = False - elif mhost == 'any' or mhost == rhost or \ + elif mhost == '*' or mhost == rhost or \ (mhost[0] == '.' and rhost.endswith(mhost)): allow = True return allow diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index 421770a241..d26df3e44a 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -29,6 +29,9 @@ class DevAuth(object): def __init__(self, app, conf): self.app = app self.conf = conf + self.reseller_prefix = conf.get('reseller_prefix', '').strip() + if self.reseller_prefix and self.reseller_prefix[-1] != '_': + self.reseller_prefix += '_' self.auth_host = conf.get('ip', '127.0.0.1') self.auth_port = int(conf.get('port', 11000)) self.ssl = \ @@ -44,7 +47,7 @@ class DevAuth(object): """ groups = None token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) - if token: + if token and token.startswith(self.reseller_prefix): memcache_client = cache_from_env(env) key = 'devauth/%s' % token cached_auth_data = memcache_client.get(key) @@ -68,6 +71,10 @@ class DevAuth(object): env['REMOTE_USER'] = groups env['swift.authorize'] = self.authorize env['swift.clean_acl'] = clean_acl + # We know the proxy logs the token, so we augment it just a bit to also + # log the authenticated user. + user = groups and groups.split(',', 1)[0] or '' + env['HTTP_X_AUTH_TOKEN'] = '%s,%s' % (user, token) return self.app(env, start_response) def authorize(self, req): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index aea7b618bd..a578dd3d1f 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -882,6 +882,7 @@ class ContainerController(Controller): @public def PUT(self, req): """HTTP PUT request handler.""" + self.clean_acls(req) error_response = check_metadata(req, 'container') if error_response: return error_response @@ -899,7 +900,6 @@ class ContainerController(Controller): self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} - self.clean_acls(req) headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -948,6 +948,7 @@ class ContainerController(Controller): @public def POST(self, req): """HTTP POST request handler.""" + self.clean_acls(req) error_response = check_metadata(req, 'container') if error_response: return error_response @@ -958,7 +959,6 @@ class ContainerController(Controller): self.account_name, self.container_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'x-cf-trans-id': self.trans_id} - self.clean_acls(req) headers.update(value for value in req.headers.iteritems() if value[0].lower() in self.pass_through_headers or value[0].lower().startswith('x-container-meta-')) @@ -1273,12 +1273,14 @@ class BaseApplication(object): # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) - if resp: - if not getattr(handler, 'delay_denial', None) and \ - 'swift.authorize' in req.environ: - return resp - else: + if not resp: + # No resp means authorized, no delayed recheck required. del req.environ['swift.authorize'] + else: + # Response indicates denial, but we might delay the denial + # and recheck later. If not delayed, return the error now. + if not getattr(handler, 'delay_denial', None): + return resp return handler(req) except Exception: self.logger.exception('ERROR Unhandled exception in request') @@ -1331,8 +1333,7 @@ class Application(BaseApplication): status_int, req.referer or '-', req.user_agent or '-', - '%s:%s' % (req.remote_user or '', - req.headers.get('x-auth-token', '-')), + req.headers.get('x-auth-token', '-'), getattr(req, 'bytes_transferred', 0) or '-', getattr(response, 'bytes_transferred', 0) or '-', req.headers.get('etag', '-'), diff --git a/test/functional/sample.conf b/test/functional/sample.conf index 983f2cf768..646541e905 100644 --- a/test/functional/sample.conf +++ b/test/functional/sample.conf @@ -2,6 +2,7 @@ auth_host = 127.0.0.1 auth_port = 11000 auth_ssl = no +auth_prefix = AUTH # Primary functional test account (needs admin access to the account) account = test diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py index 8bd46b462b..974879cc09 100644 --- a/test/functionalnosetests/swift_testing.py +++ b/test/functionalnosetests/swift_testing.py @@ -10,6 +10,9 @@ from swift.common.client import get_auth, http_connection swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') +swift_test_auth_prefix = os.environ.get('SWIFT_TEST_AUTH_PREFIX') +if swift_test_auth_prefix and swift_test_auth_prefix[-1] != '_': + swift_test_auth_prefix += '_' swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] @@ -32,6 +35,9 @@ if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]): if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): swift_test_auth = 'https' swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf + swift_test_auth_prefix = conf.get('auth_prefix', 'AUTH') + if swift_test_auth_prefix and swift_test_auth_prefix[-1] != '_': + swift_test_auth_prefix += '_' swift_test_user[0] = '%(account)s:%(username)s' % conf swift_test_key[0] = conf['password'] try: diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py index 6498f56423..c1dca1cb6b 100755 --- a/test/functionalnosetests/test_container.py +++ b/test/functionalnosetests/test_container.py @@ -9,7 +9,7 @@ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift_testing import check_response, retry, skip, skip2, skip3, \ - swift_test_user + swift_test_auth_prefix, swift_test_user class TestContainer(unittest.TestCase): @@ -334,7 +334,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Read': '.ref:any'}) + 'X-Container-Read': '.r:*'}) return check_response(conn) resp = retry(post) resp.read() @@ -377,10 +377,10 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # Make the container accessible by the second account def post(url, token, parsed, conn): - conn.request('POST', parsed.path + '/' + self.name, '', - {'X-Auth-Token': token, - 'X-Container-Read': 'test2', - 'X-Container-Write': 'test2'}) + conn.request('POST', parsed.path + '/' + self.name, '', { + 'X-Auth-Token': token, + 'X-Container-Read': '%stest2' % swift_test_auth_prefix, + 'X-Container-Write': '%stest2' % swift_test_auth_prefix}) return check_response(conn) resp = retry(post) resp.read() @@ -427,7 +427,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Read': '.ref:any'}) + 'X-Container-Read': '.r:*'}) return check_response(conn) resp = retry(post) resp.read() @@ -446,9 +446,9 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # Now make the container also writeable by the second account def post(url, token, parsed, conn): - conn.request('POST', parsed.path + '/' + self.name, '', - {'X-Auth-Token': token, - 'X-Container-Write': 'test2'}) + conn.request('POST', parsed.path + '/' + self.name, '', { + 'X-Auth-Token': token, + 'X-Container-Write': '%stest2' % swift_test_auth_prefix}) return check_response(conn) resp = retry(post) resp.read() @@ -485,7 +485,9 @@ class TestContainer(unittest.TestCase): # Make the container accessible by the third account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', - {'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]}) + {'X-Auth-Token': token, + 'X-Container-Read': '%s%s' % + (swift_test_auth_prefix, swift_test_user[2])}) return check_response(conn) resp = retry(post) resp.read() @@ -506,7 +508,8 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Write': swift_test_user[2]}) + 'X-Container-Write': '%s%s' % + (swift_test_auth_prefix, swift_test_user[2])}) return check_response(conn) resp = retry(post) resp.read() diff --git a/test/functionalnosetests/test_object.py b/test/functionalnosetests/test_object.py index 8c17b336a5..4a12bf088f 100644 --- a/test/functionalnosetests/test_object.py +++ b/test/functionalnosetests/test_object.py @@ -65,7 +65,7 @@ class TestObject(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.container, '', {'X-Auth-Token': token, - 'X-Container-Read': '.ref:any'}) + 'X-Container-Read': '.r:*'}) return check_response(conn) resp = retry(post) resp.read() diff --git a/test/unit/common/middleware/test_acl.py b/test/unit/common/middleware/test_acl.py index 26838f78df..03e76ce621 100644 --- a/test/unit/common/middleware/test_acl.py +++ b/test/unit/common/middleware/test_acl.py @@ -21,56 +21,71 @@ from swift.common.middleware import acl class TestACL(unittest.TestCase): def test_clean_acl(self): - value = acl.clean_acl('header', '.ref:any') - self.assertEquals(value, '.ref:any') - value = acl.clean_acl('header', '.ref:specific.host') - self.assertEquals(value, '.ref:specific.host') - value = acl.clean_acl('header', '.ref:.ending.with') - self.assertEquals(value, '.ref:.ending.with') - value = acl.clean_acl('header', '.ref:one,.ref:two') - self.assertEquals(value, '.ref:one,.ref:two') - value = acl.clean_acl('header', '.ref:any,.ref:-specific.host') - self.assertEquals(value, '.ref:any,.ref:-specific.host') - value = acl.clean_acl('header', '.ref:any,.ref:-.ending.with') - self.assertEquals(value, '.ref:any,.ref:-.ending.with') - value = acl.clean_acl('header', '.ref:one,.ref:-two') - self.assertEquals(value, '.ref:one,.ref:-two') - value = acl.clean_acl('header', - '.ref:one,.ref:-two,account,account:user') - self.assertEquals(value, '.ref:one,.ref:-two,account,account:user') + value = acl.clean_acl('header', '.r:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.r:specific.host') + self.assertEquals(value, '.r:specific.host') + value = acl.clean_acl('header', '.r:.ending.with') + self.assertEquals(value, '.r:.ending.with') + value = acl.clean_acl('header', '.r:*.ending.with') + self.assertEquals(value, '.r:.ending.with') + value = acl.clean_acl('header', '.r:-*.ending.with') + self.assertEquals(value, '.r:-.ending.with') + value = acl.clean_acl('header', '.r:one,.r:two') + self.assertEquals(value, '.r:one,.r:two') + value = acl.clean_acl('header', '.r:*,.r:-specific.host') + self.assertEquals(value, '.r:*,.r:-specific.host') + value = acl.clean_acl('header', '.r:*,.r:-.ending.with') + self.assertEquals(value, '.r:*,.r:-.ending.with') + value = acl.clean_acl('header', '.r:one,.r:-two') + self.assertEquals(value, '.r:one,.r:-two') + value = acl.clean_acl('header', '.r:one,.r:-two,account,account:user') + self.assertEquals(value, '.r:one,.r:-two,account,account:user') + value = acl.clean_acl('header', 'TEST_account') + self.assertEquals(value, 'TEST_account') + value = acl.clean_acl('header', '.ref:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.referer:*') + self.assertEquals(value, '.r:*') + value = acl.clean_acl('header', '.referrer:*') + self.assertEquals(value, '.r:*') value = acl.clean_acl('header', - ' .ref : one , ,, .ref:two , .ref : - three ') - self.assertEquals(value, '.ref:one,.ref:two,.ref:-three') - self.assertRaises(ValueError, acl.clean_acl, 'header', '.ref:') - self.assertRaises(ValueError, acl.clean_acl, 'header', ' .ref : ') + ' .r : one , ,, .r:two , .r : - three ') + self.assertEquals(value, '.r:one,.r:two,.r:-three') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.unknown:test') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:*.') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : * . ') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-*.') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r : - * . ') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', 'user , .r : ') + self.assertRaises(ValueError, acl.clean_acl, 'header', '.r:-') + self.assertRaises(ValueError, acl.clean_acl, 'header', ' .r : - ') self.assertRaises(ValueError, acl.clean_acl, 'header', - 'user , .ref : ') - self.assertRaises(ValueError, acl.clean_acl, 'header', '.ref:-') - self.assertRaises(ValueError, acl.clean_acl, 'header', ' .ref : - ') - self.assertRaises(ValueError, acl.clean_acl, 'header', - 'user , .ref : - ') - self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.ref:r') + 'user , .r : - ') + self.assertRaises(ValueError, acl.clean_acl, 'write-header', '.r:r') def test_parse_acl(self): self.assertEquals(acl.parse_acl(None), ([], [])) self.assertEquals(acl.parse_acl(''), ([], [])) - self.assertEquals(acl.parse_acl('.ref:ref1'), (['ref1'], [])) - self.assertEquals(acl.parse_acl('.ref:-ref1'), (['-ref1'], [])) + self.assertEquals(acl.parse_acl('.r:ref1'), (['ref1'], [])) + self.assertEquals(acl.parse_acl('.r:-ref1'), (['-ref1'], [])) self.assertEquals(acl.parse_acl('account:user'), ([], ['account:user'])) self.assertEquals(acl.parse_acl('account'), ([], ['account'])) - self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.ref:ref3,.ref:-ref4'), + self.assertEquals(acl.parse_acl('acc1,acc2:usr2,.r:ref3,.r:-ref4'), (['ref3', '-ref4'], ['acc1', 'acc2:usr2'])) self.assertEquals(acl.parse_acl( - 'acc1,acc2:usr2,.ref:ref3,acc3,acc4:usr4,.ref:ref5,.ref:-ref6'), + 'acc1,acc2:usr2,.r:ref3,acc3,acc4:usr4,.r:ref5,.r:-ref6'), (['ref3', 'ref5', '-ref6'], ['acc1', 'acc2:usr2', 'acc3', 'acc4:usr4'])) def test_referrer_allowed(self): self.assert_(not acl.referrer_allowed('host', None)) self.assert_(not acl.referrer_allowed('host', [])) - self.assert_(acl.referrer_allowed(None, ['any'])) - self.assert_(acl.referrer_allowed('', ['any'])) + self.assert_(acl.referrer_allowed(None, ['*'])) + self.assert_(acl.referrer_allowed('', ['*'])) self.assert_(not acl.referrer_allowed(None, ['specific.host'])) self.assert_(not acl.referrer_allowed('', ['specific.host'])) self.assert_(acl.referrer_allowed('http://www.example.com/index.html', @@ -93,7 +108,7 @@ class TestACL(unittest.TestCase): self.assert_(not acl.referrer_allowed('http://thief.example.com', ['.example.com', '-thief.example.com'])) self.assert_(not acl.referrer_allowed('http://thief.example.com', - ['any', '-thief.example.com'])) + ['*', '-thief.example.com'])) self.assert_(acl.referrer_allowed('http://www.example.com', ['.other.com', 'www.example.com'])) self.assert_(acl.referrer_allowed('http://www.example.com', @@ -104,7 +119,7 @@ class TestACL(unittest.TestCase): ['.example.com'])) self.assert_(not acl.referrer_allowed('../index.html', ['.example.com'])) - self.assert_(acl.referrer_allowed('www.example.com', ['any'])) + self.assert_(acl.referrer_allowed('www.example.com', ['*'])) if __name__ == '__main__': diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index 60451ff5d2..366492f09b 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -253,31 +253,31 @@ class TestAuth(unittest.TestCase): self.assert_(resp.startswith('403'), resp) req = Request.blank('/v1/cfa') req.remote_user = 'act:usr,act' - req.acl = '.ref:any' + req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/cfa') req.remote_user = 'act:usr,act' - req.acl = '.ref:.example.com' + req.acl = '.r:.example.com' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) req = Request.blank('/v1/cfa') req.remote_user = 'act:usr,act' - req.referrer = 'http://www.example.com/index.html' - req.acl = '.ref:.example.com' + req.referer = 'http://www.example.com/index.html' + req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/cfa') resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('401'), resp) req = Request.blank('/v1/cfa') - req.acl = '.ref:any' + req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/cfa') - req.acl = '.ref:.example.com' + req.acl = '.r:.example.com' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('401'), resp) req = Request.blank('/v1/cfa') - req.referrer = 'http://www.example.com/index.html' - req.acl = '.ref:.example.com' + req.referer = 'http://www.example.com/index.html' + req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index dfa2ef2d57..7aaee688e8 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -67,14 +67,13 @@ class TestContainerController(unittest.TestCase): self.assert_('x-container-write' not in response.headers) # Ensure POSTing acls works req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Timestamp': '1', 'X-Container-Read': '.ref:any', + headers={'X-Timestamp': '1', 'X-Container-Read': '.r:*', 'X-Container-Write': 'account:user'}) self.controller.POST(req) req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'HEAD'}) response = self.controller.HEAD(req) self.assert_(response.status.startswith('204')) - self.assertEquals(response.headers.get('x-container-read'), - '.ref:any') + self.assertEquals(response.headers.get('x-container-read'), '.r:*') self.assertEquals(response.headers.get('x-container-write'), 'account:user') # Ensure we can clear acls on POST @@ -89,14 +88,13 @@ class TestContainerController(unittest.TestCase): self.assert_('x-container-write' not in response.headers) # Ensure PUTing acls works req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Timestamp': '4', 'X-Container-Read': '.ref:any', + headers={'X-Timestamp': '4', 'X-Container-Read': '.r:*', 'X-Container-Write': 'account:user'}) self.controller.PUT(req) req = Request.blank('/sda1/p/a/c2', environ={'REQUEST_METHOD': 'HEAD'}) response = self.controller.HEAD(req) self.assert_(response.status.startswith('204')) - self.assertEquals(response.headers.get('x-container-read'), - '.ref:any') + self.assertEquals(response.headers.get('x-container-read'), '.r:*') self.assertEquals(response.headers.get('x-container-write'), 'account:user') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 941a3e90a1..face9d3787 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -1962,7 +1962,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Container-Read': '.ref:any'}) + headers={'X-Container-Read': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) res = controller.POST(req) @@ -1973,7 +1973,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'POST'}, - headers={'X-Container-Write': '.ref:any'}) + headers={'X-Container-Write': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) res = controller.POST(req) @@ -1989,7 +1989,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Container-Read': '.ref:any'}) + headers={'X-Container-Read': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) res = controller.PUT(req) @@ -2000,7 +2000,7 @@ class TestContainerController(unittest.TestCase): controller = proxy_server.ContainerController(self.app, 'account', 'container') req = Request.blank('/a/c', environ={'REQUEST_METHOD': 'PUT'}, - headers={'X-Container-Write': '.ref:any'}) + headers={'X-Container-Write': '.r:*'}) req.environ['swift.clean_acl'] = clean_acl self.app.update_request(req) res = controller.PUT(req) From a5df15005b38879d85ef6806f1273e19fe84f25a Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 9 Sep 2010 10:24:25 -0700 Subject: [PATCH 13/16] Got rid of inter-reseller ACLs. Enforce ACLs to only work within a reseller space. Updated docs and tests. We can expand to inter-reseller in the future with ACLs like .x:RESELLER_group --- doc/source/development_auth.rst | 22 ++++--- etc/proxy-server.conf-sample | 10 +-- swift/auth/server.py | 4 +- swift/common/middleware/auth.py | 4 +- test/functional/sample.conf | 1 - test/functionalnosetests/swift_testing.py | 6 -- test/functionalnosetests/test_container.py | 21 +++--- test/unit/common/middleware/test_auth.py | 75 ++++++++++++---------- 8 files changed, 71 insertions(+), 72 deletions(-) diff --git a/doc/source/development_auth.rst b/doc/source/development_auth.rst index 51741dcca5..bccb6e7df5 100644 --- a/doc/source/development_auth.rst +++ b/doc/source/development_auth.rst @@ -35,14 +35,15 @@ string. If the user does not have admin access to the account, the third group will be omitted. It is highly recommended that authentication server implementers prefix their -group names and tokens with a configurable reseller prefix (`AUTH_` by default -with the included DevAuth). This prefix will allow deconflicting with other -authentication servers that might be using the same Swift cluster. +tokens and Swift storage accounts they create with a configurable reseller +prefix (`AUTH_` by default with the included DevAuth). This prefix will allow +deconflicting with other authentication servers that might be using the same +Swift cluster. Otherwise, the Swift cluster will have to try all the resellers +until one validates a token or all fail. -The only other restriction is that no group name should begin with a period '.' -as that is reserved for internal Swift use (such as the .r for referrer -designations as you'll see later). This shouldn't be an issue if a reseller -prefix is in use and does not begin with a period. +A restriction with group names is that no group name should begin with a period +'.' as that is reserved for internal Swift use (such as the .r for referrer +designations as you'll see later). Example Authentication with DevAuth: @@ -54,10 +55,11 @@ Example Authentication with DevAuth: it matches the "tester" user within the "test" account for the storage account "AUTH_storage_xyz". * The external DevAuth server responds with "X-Auth-Groups: - AUTH_test:tester,AUTH_test,AUTH_storage_xyz" + test:tester,test,AUTH_storage_xyz" * Now this user will have full access (via authorization procedures later) - to the AUTH_storage_xyz Swift storage account and access to anything with - ACLs specifying at least one of those three groups returned. + to the AUTH_storage_xyz Swift storage account and access to other storage + accounts with the same `AUTH_` reseller prefix and has an ACL specifying + at least one of those three groups returned. Authorization is performed through callbacks by the Swift Proxy server to the WSGI environment's swift.authorize value, if one is set. The swift.authorize diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 6ee36451ab..c3766bfd5d 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -37,10 +37,12 @@ use = egg:swift#proxy [filter:auth] use = egg:swift#auth -# The reseller prefix, if set, will verify a token begins with this prefix -# before even attempting to validate it with the external reseller. Usefull if -# multiple auth systems are in use for one Swift cluster. -# reseller_prefix = +# The reseller prefix will verify a token begins with this prefix before even +# attempting to validate it with the external authentication server. Also, with +# authorization, only Swift storage accounts with this prefix will be +# authorized by this middleware. Useful if multiple auth systems are in use for +# one Swift cluster. +# reseller_prefix = AUTH # ip = 127.0.0.1 # port = 11000 # ssl = false diff --git a/swift/auth/server.py b/swift/auth/server.py index 931bbb6080..38c72114e5 100644 --- a/swift/auth/server.py +++ b/swift/auth/server.py @@ -361,9 +361,7 @@ class AuthController(object): validation = self.validate_token(token) if not validation: return HTTPNotFound() - groups = [ - '%s%s:%s' % (self.reseller_prefix, validation[1], validation[2]), - '%s%s' % (self.reseller_prefix, validation[1])] + groups = ['%s:%s' % (validation[1], validation[2]), validation[1]] if validation[3]: # admin access to a cfaccount groups.append(validation[3]) return HTTPNoContent(headers={'X-Auth-TTL': validation[0], diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py index d26df3e44a..03770175fd 100644 --- a/swift/common/middleware/auth.py +++ b/swift/common/middleware/auth.py @@ -29,7 +29,7 @@ class DevAuth(object): def __init__(self, app, conf): self.app = app self.conf = conf - self.reseller_prefix = conf.get('reseller_prefix', '').strip() + self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() if self.reseller_prefix and self.reseller_prefix[-1] != '_': self.reseller_prefix += '_' self.auth_host = conf.get('ip', '127.0.0.1') @@ -83,7 +83,7 @@ class DevAuth(object): WSGI response callable if not. """ version, account, container, obj = split_path(req.path, 1, 4, True) - if not account: + if not account or not account.startswith(self.reseller_prefix): return self.denied_response(req) if req.remote_user and account in req.remote_user.split(','): return None diff --git a/test/functional/sample.conf b/test/functional/sample.conf index 646541e905..983f2cf768 100644 --- a/test/functional/sample.conf +++ b/test/functional/sample.conf @@ -2,7 +2,6 @@ auth_host = 127.0.0.1 auth_port = 11000 auth_ssl = no -auth_prefix = AUTH # Primary functional test account (needs admin access to the account) account = test diff --git a/test/functionalnosetests/swift_testing.py b/test/functionalnosetests/swift_testing.py index 974879cc09..8bd46b462b 100644 --- a/test/functionalnosetests/swift_testing.py +++ b/test/functionalnosetests/swift_testing.py @@ -10,9 +10,6 @@ from swift.common.client import get_auth, http_connection swift_test_auth = os.environ.get('SWIFT_TEST_AUTH') -swift_test_auth_prefix = os.environ.get('SWIFT_TEST_AUTH_PREFIX') -if swift_test_auth_prefix and swift_test_auth_prefix[-1] != '_': - swift_test_auth_prefix += '_' swift_test_user = [os.environ.get('SWIFT_TEST_USER'), None, None] swift_test_key = [os.environ.get('SWIFT_TEST_KEY'), None, None] @@ -35,9 +32,6 @@ if not all([swift_test_auth, swift_test_user[0], swift_test_key[0]]): if conf.get('auth_ssl', 'no').lower() in ('yes', 'true', 'on', '1'): swift_test_auth = 'https' swift_test_auth += '://%(auth_host)s:%(auth_port)s/v1.0' % conf - swift_test_auth_prefix = conf.get('auth_prefix', 'AUTH') - if swift_test_auth_prefix and swift_test_auth_prefix[-1] != '_': - swift_test_auth_prefix += '_' swift_test_user[0] = '%(account)s:%(username)s' % conf swift_test_key[0] = conf['password'] try: diff --git a/test/functionalnosetests/test_container.py b/test/functionalnosetests/test_container.py index c1dca1cb6b..96c0be91e6 100755 --- a/test/functionalnosetests/test_container.py +++ b/test/functionalnosetests/test_container.py @@ -9,7 +9,7 @@ from swift.common.constraints import MAX_META_COUNT, MAX_META_NAME_LENGTH, \ MAX_META_OVERALL_SIZE, MAX_META_VALUE_LENGTH from swift_testing import check_response, retry, skip, skip2, skip3, \ - swift_test_auth_prefix, swift_test_user + swift_test_user class TestContainer(unittest.TestCase): @@ -377,10 +377,9 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # Make the container accessible by the second account def post(url, token, parsed, conn): - conn.request('POST', parsed.path + '/' + self.name, '', { - 'X-Auth-Token': token, - 'X-Container-Read': '%stest2' % swift_test_auth_prefix, - 'X-Container-Write': '%stest2' % swift_test_auth_prefix}) + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Read': 'test2', + 'X-Container-Write': 'test2'}) return check_response(conn) resp = retry(post) resp.read() @@ -446,9 +445,8 @@ class TestContainer(unittest.TestCase): self.assertEquals(resp.status, 403) # Now make the container also writeable by the second account def post(url, token, parsed, conn): - conn.request('POST', parsed.path + '/' + self.name, '', { - 'X-Auth-Token': token, - 'X-Container-Write': '%stest2' % swift_test_auth_prefix}) + conn.request('POST', parsed.path + '/' + self.name, '', + {'X-Auth-Token': token, 'X-Container-Write': 'test2'}) return check_response(conn) resp = retry(post) resp.read() @@ -485,9 +483,7 @@ class TestContainer(unittest.TestCase): # Make the container accessible by the third account def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', - {'X-Auth-Token': token, - 'X-Container-Read': '%s%s' % - (swift_test_auth_prefix, swift_test_user[2])}) + {'X-Auth-Token': token, 'X-Container-Read': swift_test_user[2]}) return check_response(conn) resp = retry(post) resp.read() @@ -508,8 +504,7 @@ class TestContainer(unittest.TestCase): def post(url, token, parsed, conn): conn.request('POST', parsed.path + '/' + self.name, '', {'X-Auth-Token': token, - 'X-Container-Write': '%s%s' % - (swift_test_auth_prefix, swift_test_user[2])}) + 'X-Container-Write': swift_test_user[2]}) return check_response(conn) resp = retry(post) resp.read() diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index 366492f09b..e19d6d4ca9 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -109,7 +109,7 @@ class TestAuth(unittest.TestCase): try: auth.http_connect = mock_http_connect(404) result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': FakeMemcache()}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()}, lambda x, y: None)) self.assert_(result.startswith('401'), result) finally: @@ -119,9 +119,9 @@ class TestAuth(unittest.TestCase): old_http_connect = auth.http_connect try: auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': FakeMemcache()}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()}, lambda x, y: None)) self.assert_(result.startswith('204'), result) finally: @@ -132,15 +132,15 @@ class TestAuth(unittest.TestCase): try: fake_memcache = FakeMemcache() auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, lambda x, y: None)) self.assert_(result.startswith('204'), result) auth.http_connect = mock_http_connect(404) # Should still be in memcache result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, lambda x, y: None)) self.assert_(result.startswith('204'), result) finally: @@ -151,15 +151,15 @@ class TestAuth(unittest.TestCase): try: fake_memcache = FakeMemcache() auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,cfa'}) + {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, lambda x, y: None)) self.assert_(result.startswith('204'), result) auth.http_connect = mock_http_connect(404) # Should still be in memcache, but expired result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': fake_memcache}, + 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, lambda x, y: None)) self.assert_(result.startswith('401'), result) finally: @@ -169,12 +169,12 @@ class TestAuth(unittest.TestCase): old_http_connect = auth.http_connect try: auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) - req = Request.blank('/v/a/c/o', headers={'x-auth-token': 't'}) + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + req = Request.blank('/v/a/c/o', headers={'x-auth-token': 'AUTH_t'}) req.environ['swift.cache'] = FakeMemcache() result = ''.join(self.test_auth(req.environ, start_response)) self.assert_(result.startswith('204'), result) - self.assertEquals(req.remote_user, 'act:usr,act,cfa') + self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') finally: auth.http_connect = old_http_connect @@ -182,7 +182,7 @@ class TestAuth(unittest.TestCase): old_http_connect = auth.http_connect try: auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) req = Request.blank('/v/a/c/o') req.environ['swift.cache'] = FakeMemcache() result = ''.join(self.test_auth(req.environ, start_response)) @@ -195,12 +195,13 @@ class TestAuth(unittest.TestCase): old_http_connect = auth.http_connect try: auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,cfa'}) - req = Request.blank('/v/a/c/o', headers={'x-storage-token': 't'}) + {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) + req = Request.blank('/v/a/c/o', + headers={'x-storage-token': 'AUTH_t'}) req.environ['swift.cache'] = FakeMemcache() result = ''.join(self.test_auth(req.environ, start_response)) self.assert_(result.startswith('204'), result) - self.assertEquals(req.remote_user, 'act:usr,act,cfa') + self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') finally: auth.http_connect = old_http_connect @@ -209,73 +210,81 @@ class TestAuth(unittest.TestCase): resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('401'), resp) req = Request.blank('/badpath') - req.remote_user = 'act:usr,act,cfa' + req.remote_user = 'act:usr,act,AUTH_cfa' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) def test_authorize_account_access(self): - req = Request.blank('/v1/cfa') - req.remote_user = 'act:usr,act,cfa' + req = Request.blank('/v1/AUTH_cfa') + req.remote_user = 'act:usr,act,AUTH_cfa' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) def test_authorize_acl_group_access(self): - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act2' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr2' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) + def test_deny_cross_reseller(self): + # Tests that cross-reseller is denied, even if ACLs/group names match + req = Request.blank('/v1/OTHER_cfa') + req.remote_user = 'act:usr,act,AUTH_cfa' + req.acl = 'act' + resp = str(self.test_auth.authorize(req)) + self.assert_(resp.startswith('403'), resp) + def test_authorize_acl_referrer_access(self): - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:.example.com' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('403'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('401'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:.example.com' resp = str(self.test_auth.authorize(req)) self.assert_(resp.startswith('401'), resp) - req = Request.blank('/v1/cfa') + req = Request.blank('/v1/AUTH_cfa') req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) From c931d7602a233fad38b6e69466012eff1174fcb9 Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 9 Sep 2010 10:28:50 -0700 Subject: [PATCH 14/16] Updated included client.py in st --- bin/st | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/bin/st b/bin/st index 7d672b1eb9..7c327e3943 100755 --- a/bin/st +++ b/bin/st @@ -513,10 +513,6 @@ except: raise ClientException('Object GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) if resp_chunk_size: def _object_body(): @@ -559,10 +555,6 @@ except: raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, http_host=conn.host, http_port=conn.port, http_path=path, http_status=resp.status, http_reason=resp.reason) - metadata = {} - for key, value in resp.getheaders(): - if key.lower().startswith('x-object-meta-'): - metadata[unquote(key[len('x-object-meta-'):])] = unquote(value) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value @@ -635,12 +627,12 @@ except: def post_object(url, token, container, name, headers, http_conn=None): """ - Change object metadata + Update object metadata :param url: storage URL :param token: auth token :param container: container name that the object is in - :param name: object name to change + :param name: name of the object to update :param headers: additional headers to include in the request :param http_conn: HTTP connection object (If None, it will create the conn object) From 6b353a520f05056d1cb136b9d674109de33f719f Mon Sep 17 00:00:00 2001 From: gholt Date: Thu, 9 Sep 2010 10:42:41 -0700 Subject: [PATCH 15/16] Fix bug on auto-container create during upload. --- bin/st | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/st b/bin/st index 7c327e3943..51a7637881 100755 --- a/bin/st +++ b/bin/st @@ -1354,8 +1354,15 @@ def st_upload(options, args): for thread in file_threads: thread.start() conn = create_connection() + # Try to create the container, just in case it doesn't exist. If this + # fails, it might just be because the user doesn't have container PUT + # permissions, so we'll ignore any error. If there's really a problem, + # it'll surface on the first object PUT. try: conn.put_container(args[0]) + except: + pass + try: for arg in args[1:]: if isdir(arg): _upload_dir(arg) From 7057a7c3b3897208cf85bf089d6eafd1e9105c94 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Sep 2010 07:52:10 -0700 Subject: [PATCH 16/16] Proof that test "coverage" != no obvious bugs --- swift/proxy/server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index a578dd3d1f..892c4f4769 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -851,6 +851,7 @@ class ContainerController(Controller): req.headers[header]) except ValueError, err: return HTTPBadRequest(request=req, body=str(err)) + return None def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" @@ -882,8 +883,8 @@ class ContainerController(Controller): @public def PUT(self, req): """HTTP PUT request handler.""" - self.clean_acls(req) - error_response = check_metadata(req, 'container') + error_response = \ + self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH: @@ -948,8 +949,8 @@ class ContainerController(Controller): @public def POST(self, req): """HTTP POST request handler.""" - self.clean_acls(req) - error_response = check_metadata(req, 'container') + error_response = \ + self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response account_partition, accounts = self.account_info(self.account_name)