openstack/swift
Code Issues Proposed changes

Merge from trunk

Browse Source
This commit is contained in:
gholt 2011-01-25 15:24:09 -08:00
commit d41e77417c
13 changed files with 77 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
AUTHORS
View File

@ -24,9 +24,13 @@ Paul Jimenez
Brian K. Jones
Ed Leafe
Stephen Milton
Russ Nelson
Colin Nicholson
Andrew Clay Shafer
Monty Taylor
Caleb Tennis
FUJITA Tomonori
Kapil Thangavelu
Conrad Weidenkeller
Chris Wedgwood
Cory Wright

2
swift/__init__.py
View File

@ -1,5 +1,5 @@
import gettext
__version__ = '1.1.0'
__version__ = '1.2.0'
gettext.install('swift')

60
swift/auth/server.py
View File

@ -149,31 +149,32 @@ class AuthController(object):
previous_prefix = ''
if '_' in row[0]:
previous_prefix = row[0].split('_', 1)[0]
msg = _(('''
msg = (_('''
THERE ARE ACCOUNTS IN YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW RESELLER
PREFIX OF "%s".
PREFIX OF "%(reseller)s".
YOU HAVE A FEW OPTIONS:
1) RUN "swift-auth-update-reseller-prefixes %s %s",
1. RUN "swift-auth-update-reseller-prefixes %(db_file)s %(reseller)s",
"swift-init auth-server restart", AND
"swift-auth-recreate-accounts -K ..." TO CREATE FRESH ACCOUNTS.
OR
2) REMOVE %s, RUN "swift-init auth-server restart", AND RUN
2. REMOVE %(db_file)s, RUN "swift-init auth-server restart", AND RUN
"swift-auth-add-user ..." TO CREATE BRAND NEW ACCOUNTS THAT WAY.
OR
3) ADD "reseller_prefix = %s" (WITHOUT THE QUOTES) TO YOUR
3. ADD "reseller_prefix = %(previous)s" (WITHOUT THE QUOTES) TO YOUR
proxy-server.conf IN THE [filter:auth] SECTION AND TO YOUR
auth-server.conf IN THE [app:auth-server] SECTION AND RUN
"swift-init proxy-server restart" AND "swift-init auth-server restart"
TO REVERT BACK TO YOUR PREVIOUS RESELLER PREFIX.
%s
''') % (self.reseller_prefix.rstrip('_'), self.db_file,
self.reseller_prefix.rstrip('_'), self.db_file,
previous_prefix, previous_prefix and ' ' or _('''
%(note)s
''') % {'reseller': self.reseller_prefix.rstrip('_'),
'db_file': self.db_file,
'previous': previous_prefix,
'note': previous_prefix and ' ' or _('''
SINCE YOUR PREVIOUS RESELLER PREFIX WAS AN EMPTY STRING, IT IS NOT
RECOMMENDED TO PERFORM OPTION 3 AS THAT WOULD MAKE SUPPORTING MULTIPLE
RESELLERS MORE DIFFICULT.
''').strip())).strip()
''').strip()}).strip()
self.logger.critical(_('CRITICAL: ') + ' '.join(msg.split()))
raise Exception('\n' + msg)
@ -243,7 +244,8 @@ YOU HAVE A FEW OPTIONS:
raise err
def validate_s3_sign(self, request, token):
account, user, sign = request.headers['Authorization'].split(' ')[-1].split(':')
account, user, sign = \
request.headers['Authorization'].split(' ')[-1].split(':')
msg = base64.urlsafe_b64decode(unquote(token))
rv = False
with self.get_conn() as conn:
@ -253,7 +255,8 @@ YOU HAVE A FEW OPTIONS:
(account, user)).fetchone()
rv = (84000, account, user, row[1])
if rv:
s = base64.encodestring(hmac.new(row[0], msg, sha1).digest()).strip()
s = base64.encodestring(hmac.new(row[0], msg,
sha1).digest()).strip()
self.logger.info("orig %s, calc %s" % (sign, s))
if sign != s:
rv = False
@ -340,10 +343,14 @@ YOU HAVE A FEW OPTIONS:
'SELECT url FROM account WHERE account = ? AND user = ?',
(account, user)).fetchone()
if row:
self.logger.info(
_('ALREADY EXISTS create_user(%s, %s, _, %s, %s) [%.02f]') %
(repr(account), repr(user), repr(admin),
repr(reseller_admin), time() - begin))
self.logger.info(_('ALREADY EXISTS create_user(%(account)s, '
'%(user)s, _, %(admin)s, %(reseller_admin)s) '
'[%(elapsed).02f]') %
{'account': repr(account),
'user': repr(user),
'admin': repr(admin),
'reseller_admin': repr(reseller_admin),
'elapsed': time() - begin})
return 'already exists'
row = conn.execute(
'SELECT url, cfaccount FROM account WHERE account = ?',
@ -354,10 +361,14 @@ YOU HAVE A FEW OPTIONS:
else:
account_hash = self.add_storage_account()
if not account_hash:
self.logger.info(
_('FAILED create_user(%s, %s, _, %s, %s) [%.02f]') %
(repr(account), repr(user), repr(admin),
repr(reseller_admin), time() - begin))
self.logger.info(_('FAILED create_user(%(account)s, '
'%(user)s, _, %(admin)s, %(reseller_admin)s) '
'[%(elapsed).02f]') %
{'account': repr(account),
'user': repr(user),
'admin': repr(admin),
'reseller_admin': repr(reseller_admin),
'elapsed': time() - begin})
return False
url = self.default_cluster_url.rstrip('/') + '/' + account_hash
conn.execute('''INSERT INTO account
@ -367,10 +378,11 @@ YOU HAVE A FEW OPTIONS:
(account, url, account_hash, user, password,
admin and 't' or '', reseller_admin and 't' or ''))
conn.commit()
self.logger.info(
_('SUCCESS create_user(%s, %s, _, %s, %s) = %s [%.02f]') %
(repr(account), repr(user), repr(admin), repr(reseller_admin),
repr(url), time() - begin))
self.logger.info(_('SUCCESS create_user(%(account)s, %(user)s, _, '
'%(admin)s, %(reseller_admin)s) = %(url)s [%(elapsed).02f]') %
{'account': repr(account), 'user': repr(user),
'admin': repr(admin), 'reseller_admin': repr(reseller_admin),
'url': repr(url), 'elapsed': time() - begin})
return url
def recreate_accounts(self):

10
swift/common/middleware/auth.py
View File

@ -59,8 +59,8 @@ class DevAuth(object):
if s3 or (token and token.startswith(self.reseller_prefix)):
# Note: Empty reseller_prefix will match all tokens.
# Attempt to auth my token with my auth server
groups = \
self.get_groups(env, token, memcache_client=cache_from_env(env))
groups = self.get_groups(env, token,
memcache_client=cache_from_env(env))
if groups:
env['REMOTE_USER'] = groups
user = groups and groups.split(',', 1)[0] or ''
@ -154,10 +154,12 @@ class DevAuth(object):
timeout=expiration)
if env.get('HTTP_AUTHORIZATION'):
account, user, sign = env['HTTP_AUTHORIZATION'].split(' ')[-1].split(':')
account, user, sign = \
env['HTTP_AUTHORIZATION'].split(' ')[-1].split(':')
cfaccount = resp.getheader('x-auth-account-suffix')
path = env['PATH_INFO']
env['PATH_INFO'] = path.replace("%s:%s" % (account, user), cfaccount, 1)
env['PATH_INFO'] = \
path.replace("%s:%s" % (account, user), cfaccount, 1)
return groups

3
swift/common/middleware/domain_remap.py
View File

@ -35,7 +35,7 @@ class DomainRemapMiddleware(object):
instead of the found reseller prefix. The reseller_prefixes list is
exclusive. If defined, any request with an account prefix not in that list
will be ignored by this middleware. reseller_prefixes defaults to 'AUTH'.
Note that this middleware requires that container names and account names
(except as described above) must be DNS-compatible. This means that the
account name created in the system and the containers created by users
@ -111,4 +111,3 @@ def filter_factory(global_conf, **local_conf):
def domain_filter(app):
return DomainRemapMiddleware(app, conf)
return domain_filter

3
swift/obj/replicator.py
View File

@ -407,7 +407,8 @@ class ObjectReplicator(Daemon):
conn.getresponse().read()
self.suffix_sync += len(suffixes)
except (Exception, Timeout):
self.logger.exception(_("Error syncing with node: %s") % node)
self.logger.exception(_("Error syncing with node: %s") %
node)
self.suffix_count += len(local_hash)
except (Exception, Timeout):
self.logger.exception(_("Error syncing partition"))

7
swift/stats/account_stats.py
View File

@ -55,7 +55,8 @@ class AccountStat(Daemon):
self.logger.info(_("Gathering account stats"))
start = time.time()
self.find_and_process()
self.logger.info(_("Gathering account stats complete (%0.2f minutes)") %
self.logger.info(
_("Gathering account stats complete (%0.2f minutes)") %
((time.time() - start) / 60))
def find_and_process(self):
@ -70,8 +71,8 @@ class AccountStat(Daemon):
# Account Name, Container Count, Object Count, Bytes Used
for device in os.listdir(self.devices):
if self.mount_check and not check_mount(self.devices, device):
self.logger.error(_("Device %s is not mounted, skipping.") %
device)
self.logger.error(
_("Device %s is not mounted, skipping.") % device)
continue
accounts = os.path.join(self.devices,
device,

3
swift/stats/log_processor.py
View File

@ -280,7 +280,8 @@ class LogProcessorDaemon(Daemon):
logs_to_process = self.log_processor.get_data_list(lookback_start,
lookback_end,
already_processed_files)
self.logger.info(_('loaded %d files to process') % len(logs_to_process))
self.logger.info(_('loaded %d files to process') %
len(logs_to_process))
if not logs_to_process:
self.logger.info(_("Log processing done (%0.2f minutes)") %
((time.time() - start) / 60))

2
test/unit/container/test_server.py
View File

@ -45,7 +45,7 @@ class TestContainerController(unittest.TestCase):
def tearDown(self):
""" Tear down for testing swift.object_server.ObjectController """
rmtree(self.testdir, ignore_errors=1)
rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_acl_container(self):
# Ensure no acl by default

2
test/unit/container/test_updater.py
View File

@ -51,7 +51,7 @@ class TestContainerUpdater(unittest.TestCase):
os.mkdir(self.sda1)
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_creation(self):
cu = container_updater.ContainerUpdater({

30
test/unit/obj/test_auditor.py
View File

@ -56,7 +56,7 @@ class TestAuditor(unittest.TestCase):
mount_check='false')
def tearDown(self):
rmtree(self.testdir, ignore_errors=1)
rmtree(os.path.dirname(self.testdir), ignore_errors=1)
def test_object_audit_extra_data(self):
self.auditor = auditor.ObjectAuditor(self.conf)
@ -123,25 +123,21 @@ class TestAuditor(unittest.TestCase):
self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def test_object_audit_no_meta(self):
self.auditor = auditor.ObjectAuditor(self.conf)
cur_part = '0'
disk_file = DiskFile(self.devices, 'sda', cur_part, 'a', 'c', 'o')
data = '0' * 1024
etag = md5()
timestamp = str(normalize_timestamp(time.time()))
path = os.path.join(disk_file.datadir, timestamp + '.data')
mkdirs(disk_file.datadir)
fp = open(path, 'w')
fp.write('0' * 1024)
fp.close()
invalidate_hash(os.path.dirname(disk_file.datadir))
self.auditor = auditor.ObjectAuditor(self.conf)
pre_quarantines = self.auditor.quarantines
with disk_file.mkstemp() as (fd, tmppath):
os.write(fd, data)
etag.update(data)
etag = etag.hexdigest()
timestamp = str(normalize_timestamp(time.time()))
os.fsync(fd)
invalidate_hash(os.path.dirname(disk_file.datadir))
renamer(tmppath, os.path.join(disk_file.datadir,
timestamp + '.data'))
self.auditor.object_audit(
os.path.join(disk_file.datadir, timestamp + '.data'),
'sda', cur_part)
self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
self.auditor.object_audit(
os.path.join(disk_file.datadir, timestamp + '.data'),
'sda', cur_part)
self.assertEquals(self.auditor.quarantines, pre_quarantines + 1)
def test_object_audit_bad_args(self):
self.auditor = auditor.ObjectAuditor(self.conf)

2
test/unit/obj/test_server.py
View File

@ -53,7 +53,7 @@ class TestObjectController(unittest.TestCase):
def tearDown(self):
""" Tear down for testing swift.object_server.ObjectController """
rmtree(self.testdir)
rmtree(os.path.dirname(self.testdir))
def test_POST_update_meta(self):
""" Test swift.object_server.ObjectController.POST """

8
test/unit/proxy/test_server.py
View File

@ -142,7 +142,7 @@ def teardown():
for server in _test_coros:
server.kill()
proxy_server.CONTAINER_LISTING_LIMIT = _orig_container_listing_limit
rmtree(_testdir)
rmtree(os.path.dirname(_testdir))
def fake_http_connect(*code_iter, **kwargs):
@ -3425,5 +3425,7 @@ class TestSegmentedIterable(unittest.TestCase):
if __name__ == '__main__':
setup()
unittest.main()
teardown()
try:
unittest.main()
finally:
teardown()