openstack/swift
Code Issues Proposed changes

Added account_autocreate mode where authorized requests to accounts that don't yet exist within Swift will cause those accounts to be automatically created. Also did a small refactor surrounding swift.common.utils.TRUE_VALUES.

Browse Source
This commit is contained in:
gholt 2011-06-10 22:27:05 +00:00 committed by Tarmac
commit 156b6e0c92
10 changed files with 92 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/source/deployment_guide.rst
View File

@ -547,6 +547,10 @@ error_suppression_limit 10 Error count to consider a
node error limited
allow_account_management false Whether account PUTs and DELETEs
are even callable
account_autocreate false If set to 'true' authorized
accounts that do not yet exist
within the Swift cluster will
be automatically created.
============================ =============== =============================
[tempauth]

3
etc/proxy-server.conf-sample
View File

@ -40,6 +40,9 @@ use = egg:swift#proxy
# If set to 'true' any authorized user may create and delete accounts; if
# 'false' no one, even authorized, can.
# allow_account_management = false
# If set to 'true' authorized accounts that do not yet exist within the Swift
# cluster will be automatically created.
# account_autocreate = false
[filter:tempauth]
use = egg:swift#tempauth

4
swift/common/bench.py
View File

@ -43,7 +43,7 @@ class Bench(object):
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = conf.use_proxy in TRUE_VALUES
self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES
if self.use_proxy:
url, token = client.get_auth(self.auth_url, self.user, self.key)
self.token = token
@ -125,7 +125,7 @@ class BenchController(object):
self.logger = logger
self.conf = conf
self.names = []
self.delete = conf.delete in TRUE_VALUES
self.delete = conf.delete.lower() in TRUE_VALUES
self.gets = int(conf.num_gets)
def run(self):

3
swift/common/daemon.py
View File

@ -75,7 +75,8 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
log_name=kwargs.get('log_name'))
# once on command line (i.e. daemonize=false) will over-ride config
once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
once = once or \
conf.get('daemonize', 'true').lower() not in utils.TRUE_VALUES
# pre-configure logger
if 'logger' in kwargs:

2
swift/common/middleware/staticweb.py
View File

@ -270,7 +270,7 @@ class StaticWeb(object):
:param start_response: The original WSGI start_response hook.
:param prefix: Any prefix desired for the container listing.
"""
if self._listings not in TRUE_VALUES:
if self._listings.lower() not in TRUE_VALUES:
resp = HTTPNotFound()(env, self._start_response)
return self._error_response(resp, env, start_response)
tmp_env = self._get_escalated_env(env)

2
swift/common/utils.py
View File

@ -72,7 +72,7 @@ if hash_conf.read('/etc/swift/swift.conf'):
pass
# Used when reading config values
TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On', 't', 'y'))
TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
def validate_configuration():

41
swift/proxy/server.py
View File

@ -41,8 +41,8 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \
from webob import Request, Response
from swift.common.ring import Ring
from swift.common.utils import get_logger, normalize_timestamp, split_path, \
cache_from_env, ContextPool
from swift.common.utils import cache_from_env, ContextPool, get_logger, \
normalize_timestamp, split_path, TRUE_VALUES
from swift.common.bufferedhttp import http_connect
from swift.common.constraints import check_metadata, check_object_creation, \
check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \
@ -338,7 +338,7 @@ class Controller(object):
node['errors'] = self.app.error_suppression_limit + 1
node['last_error'] = time.time()
def account_info(self, account):
def account_info(self, account, autocreate=False):
"""
Get account information, and also verify that the account exists.
@ -353,7 +353,7 @@ class Controller(object):
result_code = self.app.memcache.get(cache_key)
if result_code == 200:
return partition, nodes
elif result_code == 404:
elif result_code == 404 and not autocreate:
return None, None
result_code = 0
attempts_left = self.app.account_ring.replica_count
@ -386,6 +386,17 @@ class Controller(object):
except (Exception, TimeoutError):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s') % path)
if result_code == 404 and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id}
resp = self.make_requests(Request.blank('/v1' + path),
self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes))
if resp.status_int // 100 != 2:
raise Exception('Could not autocreate account %r' % path)
result_code = 200
if self.app.memcache and result_code in (200, 404):
if result_code == 200:
cache_timeout = self.app.recheck_account_existence
@ -397,7 +408,7 @@ class Controller(object):
return partition, nodes
return None, None
def container_info(self, account, container):
def container_info(self, account, container, account_autocreate=False):
"""
Get container information and thusly verify container existance.
This will also make a call to account_info to verify that the
@ -423,7 +434,7 @@ class Controller(object):
return partition, nodes, read_acl, write_acl
elif status == 404:
return None, None, None, None
if not self.account_info(account)[1]:
if not self.account_info(account, autocreate=account_autocreate)[1]:
return None, None, None, None
result_code = 0
read_acl = None
@ -854,7 +865,8 @@ class ObjectController(Controller):
if error_response:
return error_response
container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
@ -911,7 +923,8 @@ class ObjectController(Controller):
def PUT(self, req):
"""HTTP PUT request handler."""
container_partition, containers, _junk, req.acl = \
self.container_info(self.account_name, self.container_name)
self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
@ -1219,7 +1232,8 @@ class ContainerController(Controller):
resp.body = 'Container name length of %d longer than %d' % \
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return resp
account_partition, accounts = self.account_info(self.account_name)
account_partition, accounts = self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
@ -1249,7 +1263,8 @@ class ContainerController(Controller):
self.clean_acls(req) or check_metadata(req, 'container')
if error_response:
return error_response
account_partition, accounts = self.account_info(self.account_name)
account_partition, accounts = self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if not accounts:
return HTTPNotFound(request=req)
container_partition, containers = self.app.container_ring.get_nodes(
@ -1391,7 +1406,7 @@ class BaseApplication(object):
self.put_queue_depth = int(conf.get('put_queue_depth', 10))
self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.log_headers = conf.get('log_headers') == 'True'
self.log_headers = conf.get('log_headers', 'no').lower() in TRUE_VALUES
self.error_suppression_interval = \
int(conf.get('error_suppression_interval', 60))
self.error_suppression_limit = \
@ -1401,7 +1416,7 @@ class BaseApplication(object):
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
conf.get('allow_account_management', 'false').lower() == 'true'
conf.get('allow_account_management', 'no').lower() in TRUE_VALUES
self.resellers_conf = ConfigParser()
self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf'))
self.object_ring = object_ring or \
@ -1413,6 +1428,8 @@ class BaseApplication(object):
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(swift_dir, 'mime.types')])
self.account_autocreate = \
conf.get('account_autocreate', 'no').lower() in TRUE_VALUES
def get_controller(self, path):
"""

2
swift/stats/log_uploader.py
View File

@ -69,7 +69,7 @@ class LogUploader(Daemon):
self.internal_proxy = InternalProxy(proxy_server_conf)
self.new_log_cutoff = int(cutoff or
uploader_conf.get('new_log_cutoff', '7200'))
self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \
self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \
utils.TRUE_VALUES
self.filename_pattern = regex or \
uploader_conf.get('source_filename_pattern',

4
test/unit/common/test_utils.py
View File

@ -768,6 +768,10 @@ log_name = yarr'''
self.assertEquals(utils.human_readable(1237940039285380274899124224),
'1024Yi')
def test_TRUE_VALUES(self):
for v in utils.TRUE_VALUES:
self.assertEquals(v, v.lower())
if __name__ == '__main__':
unittest.main()

48
test/unit/proxy/test_server.py
View File

@ -393,6 +393,48 @@ class TestController(unittest.TestCase):
test(404, 507, 503)
test(503, 503, 503)
def test_account_info_account_autocreate(self):
with save_globals():
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=False)
self.check_account_info_return(partition, nodes, is_none=True)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account)
self.check_account_info_return(partition, nodes, is_none=True)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 201, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 201)
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
self.check_account_info_return(partition, nodes)
self.memcache.store = {}
proxy_server.http_connect = \
fake_http_connect(404, 404, 404, 503, 201, 503)
exc = None
try:
partition, nodes = \
self.controller.account_info(self.account, autocreate=True)
except Exception, err:
exc = err
self.assertEquals(str(exc),
"Could not autocreate account '/some_account'")
def check_container_info_return(self, ret, is_none=False):
if is_none:
partition, nodes, read_acl, write_acl = None, None, None, None
@ -406,7 +448,7 @@ class TestController(unittest.TestCase):
self.assertEqual(write_acl, ret[3])
def test_container_info_invalid_account(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return None, None
with save_globals():
@ -417,7 +459,7 @@ class TestController(unittest.TestCase):
# tests if 200 is cached and used
def test_container_info_200(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return True, True
with save_globals():
@ -443,7 +485,7 @@ class TestController(unittest.TestCase):
# tests if 404 is cached and used
def test_container_info_404(self):
def account_info(self, account):
def account_info(self, account, autocreate=False):
return True, True
with save_globals():