openstack/swift
Code Issues Proposed changes

new swauth-cleanup-tokens; restricted listing .auth account to .super_admin; doc updates

Browse Source
This commit is contained in:
gholt 2010-12-09 17:57:26 -08:00
parent d13ea1dbec
commit 09e39032bf
7 changed files with 180 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
bin/swauth-cleanup-tokens Executable file
View File

@ -0,0 +1,102 @@
#!/usr/bin/python
# Copyright (c) 2010 OpenStack, LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
try:
import simplejson as json
except ImportError:
import json
import re
from datetime import datetime, timedelta
from optparse import OptionParser
from sys import argv, exit
from time import sleep, time
from swift.common.client import Connection
if __name__ == '__main__':
parser = OptionParser(usage='Usage: %prog [options]')
parser.add_option('-t', '--token-life', dest='token_life',
default='86400', help='The expected life of tokens; token objects '
'modified more than this number of seconds ago will be checked for '
'expiration (default: 86400).')
parser.add_option('-s', '--sleep', dest='sleep',
default='0.1', help='The number of seconds to sleep between token '
'checks (default: 0.1)')
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
default=False, help='Outputs everything done instead of just the '
'deletions.')
parser.add_option('-A', '--admin-url', dest='admin_url',
default='http://127.0.0.1:8080/auth/', help='The URL to the auth '
'subsystem (default: http://127.0.0.1:8080/auth/)')
parser.add_option('-K', '--admin-key', dest='admin_key',
help='The key for .super_admin.')
args = argv[1:]
if not args:
args.append('-h')
(options, args) = parser.parse_args(args)
if len(args) != 0:
parser.parse_args(['-h'])
options.admin_url = options.admin_url.rstrip('/')
if not options.admin_url.endswith('/v1.0'):
options.admin_url += '/v1.0'
options.admin_user = '.super_admin:.super_admin'
options.token_life = timedelta(0, float(options.token_life))
options.sleep = float(options.sleep)
conn = Connection(options.admin_url, options.admin_user, options.admin_key)
for x in xrange(16):
container = '.token_%x' % x
marker = None
while True:
if options.verbose:
print 'GET %s?marker=%s' % (container, marker)
objs = conn.get_container(container, marker=marker)[1]
if objs:
marker = objs[-1]['name']
else:
if options.verbose:
print 'No more objects in %s' % container
break
for obj in objs:
last_modified = datetime(*map(int, re.split('[^\d]',
obj['last_modified'])[:-1]))
ago = datetime.utcnow() - last_modified
if ago > options.token_life:
if options.verbose:
print '%s/%s last modified %ss ago; investigating' % \
(container, obj['name'],
ago.days * 86400 + ago.seconds)
print 'GET %s/%s' % (container, obj['name'])
detail = conn.get_object(container, obj['name'])[1]
detail = json.loads(detail)
if detail['expires'] < time():
if options.verbose:
print '%s/%s expired %ds ago; deleting' % \
(container, obj['name'],
time() - detail['expires'])
print 'DELETE %s/%s' % (container, obj['name'])
conn.delete_object(container, obj['name'])
elif options.verbose:
print "%s/%s won't expire for %ds; skipping" % \
(container, obj['name'],
detail['expires'] - time())
elif options.verbose:
print '%s/%s last modified %ss ago; skipping' % \
(container, obj['name'],
ago.days * 86400 + ago.seconds)
sleep(options.sleep)
if options.verbose:
print 'Done.'

15
doc/source/admin_guide.rst
View File

@ -232,6 +232,21 @@ get performance timings (warning: the initial populate takes a while). These
timings are dumped into a CSV file (/etc/swift/stats.csv by default) and can
then be graphed to see how cluster performance is trending.
------------------------------------
Additional Cleanup Script for Swauth
------------------------------------
If you decide to use Swauth, you'll want to install a cronjob to clean up any
orphaned expired tokens. These orphaned tokens can occur when a "stampede"
occurs where a single user authenticates several times concurrently. Generally,
these orphaned tokens don't pose much of an issue, but it's good to clean them
up once a "token life" period (default: 1 day or 86400 seconds).
This should be as simple as adding `swauth-cleanup-tokens -K swauthkey >
/dev/null` to a crontab entry on one of the proxies that is running Swauth; but
run `swauth-cleanup-tokens` with no arguments for detailed help on the options
available.
------------------------
Debugging Tips and Tools
------------------------

14
doc/source/howto_installmultinode.rst
View File

@ -455,6 +455,20 @@ See :ref:`config-proxy` for the initial setup, and then follow these additional
#. After you sync all the nodes, make sure the admin has the keys in /etc/swift and the ownership for the ring file is correct.
Additional Cleanup Script for Swauth
------------------------------------
If you decide to use Swauth, you'll want to install a cronjob to clean up any
orphaned expired tokens. These orphaned tokens can occur when a "stampede"
occurs where a single user authenticates several times concurrently. Generally,
these orphaned tokens don't pose much of an issue, but it's good to clean them
up once a "token life" period (default: 1 day or 86400 seconds).
This should be as simple as adding `swauth-cleanup-tokens -K swauthkey >
/dev/null` to a crontab entry on one of the proxies that is running Swauth; but
run `swauth-cleanup-tokens` with no arguments for detailed help on the options
available.
Troubleshooting Notes
---------------------
If you see problems, look in var/log/syslog (or messages on some distros).

40
doc/source/overview_auth.rst
View File

@ -78,13 +78,14 @@ objects contain a JSON dictionary of the format::
The `<auth_type>` can only be `plaintext` at this time, and the `<auth_value>`
is the plain text password itself.
The `<groups_array>` contains at least two group names. The first is a unique
group name identifying that user and is of the format `<user>:<account>`. The
The `<groups_array>` contains at least two groups. The first is a unique group
identifying that user and it's name is of the format `<user>:<account>`. The
second group is the `<account>` itself. Additional groups of `.admin` for
account administrators and `.reseller_admin` for reseller administrators may
exist. Here's an example user JSON dictionary::
{"auth": "plaintext:testing", "groups": ["test:tester", "test", ".admin"]}
{"auth": "plaintext:testing",
"groups": ["name": "test:tester", "name": "test", "name": ".admin"]}
To map an auth service account to a Swift storage account, the Service Account
Id string is stored in the `X-Container-Meta-Account-Id` header for the
@ -147,7 +148,7 @@ Here's an example token object's JSON dictionary::
{"account": "test",
"user": "tester",
"account_id": "AUTH_8980f74b1cda41e483cbe0a925f448a9",
"groups": ["test:tester", "test", ".admin"],
"groups": ["name": "test:tester", "name": "test", "name": ".admin"],
"expires": 1291273147.1624689}
To easily map a user to an already issued token, the token name is stored in
@ -156,14 +157,29 @@ the user object's `X-Object-Meta-Auth-Token` header.
Here is an example full listing of an <auth_account>::
.account_id
AUTH_4a4e6655-4c8e-4bcb-b73e-0ff1104c4fef
AUTH_5162ec51-f792-4db3-8a35-b3439a1bf6fd
AUTH_8efbea51-9339-42f8-8ac5-f26e1da67eed
.token
AUTH_tk03d8571f735a4ec9abccc704df941c6e
AUTH_tk27cf3f2029b64ec8b56c5d638807b3de
AUTH_tk7594203449754c22a34ac7d910521c2e
AUTH_tk8f2ee54605dd42a8913d244de544d19e
AUTH_2282f516-559f-4966-b239-b5c88829e927
AUTH_f6f57a3c-33b5-4e85-95a5-a801e67505c8
AUTH_fea96a36-c177-4ca4-8c7e-b8c715d9d37b
.token_0
.token_1
.token_2
.token_3
.token_4
.token_5
.token_6
AUTH_tk9d2941b13d524b268367116ef956dee6
.token_7
.token_8
AUTH_tk93627c6324c64f78be746f1e6a4e3f98
.token_9
.token_a
.token_b
.token_c
.token_d
.token_e
AUTH_tk0d37d286af2c43ffad06e99112b3ec4e
.token_f
AUTH_tk766bbde93771489982d8dc76979d11cf
reseller
.services
reseller

6
setup.py
View File

@ -81,9 +81,9 @@ setup(
'bin/swift-log-stats-collector',
'bin/swift-account-stats-logger',
'bin/swauth-add-account', 'bin/swauth-add-user',
'bin/swauth-delete-account', 'bin/swauth-delete-user',
'bin/swauth-list', 'bin/swauth-prep', 'bin/swauth-set-account-service',
'bin/swift-auth-to-swauth',
'bin/swauth-cleanup-tokens', 'bin/swauth-delete-account',
'bin/swauth-delete-user', 'bin/swauth-list', 'bin/swauth-prep',
'bin/swauth-set-account-service', 'bin/swift-auth-to-swauth',
],
entry_points={
'paste.app_factory': [

25
swift/common/middleware/swauth.py
View File

@ -174,7 +174,7 @@ class Swauth(object):
if expires < time():
groups = None
if not groups:
path = quote('/v1/%s/.token%s/%s' %
path = quote('/v1/%s/.token_%s/%s' %
(self.auth_account, token[-1], token))
resp = self.make_request(env, 'GET', path).get_response(self.app)
if resp.status_int // 100 != 2:
@ -205,7 +205,8 @@ class Swauth(object):
if not account or not account.startswith(self.reseller_prefix):
return self.denied_response(req)
user_groups = (req.remote_user or '').split(',')
if '.reseller_admin' in user_groups:
if '.reseller_admin' in user_groups and \
account[len(self.reseller_prefix)].isalnum():
return None
if account in user_groups and (req.method != 'PUT' or container):
# If the user is admin for the account and is not trying to do an
@ -339,7 +340,7 @@ class Swauth(object):
raise Exception('Could not create container: %s %s' %
(path, resp.status))
for container in xrange(16):
path = quote('/v1/%s/.token%x' % (self.auth_account, container))
path = quote('/v1/%s/.token_%x' % (self.auth_account, container))
resp = self.make_request(req.environ, 'PUT',
path).get_response(self.app)
if resp.status_int // 100 != 2:
@ -852,7 +853,7 @@ class Swauth(object):
(path, resp.status))
candidate_token = resp.headers.get('x-object-meta-auth-token')
if candidate_token:
path = quote('/v1/%s/.token%s/%s' %
path = quote('/v1/%s/.token_%s/%s' %
(self.auth_account, candidate_token[-1], candidate_token))
resp = self.make_request(req.environ, 'DELETE',
path).get_response(self.app)
@ -943,6 +944,13 @@ class Swauth(object):
return HTTPBadRequest(request=req)
if not all((account, user, key)):
return HTTPUnauthorized(request=req)
if user == '.super_admin' and key == self.super_admin_key:
token = self.get_itoken(req.environ)
url = '%s/%s.auth' % (self.dsc_url, self.reseller_prefix)
return Response(request=req,
body=json.dumps({'storage': {'default': 'local', 'local': url}}),
headers={'x-auth-token': token, 'x-storage-token': token,
'x-storage-url': url})
# Authenticate user
path = quote('/v1/%s/%s/%s' % (self.auth_account, account, user))
resp = self.make_request(req.environ, 'GET',
@ -959,7 +967,7 @@ class Swauth(object):
token = None
candidate_token = resp.headers.get('x-object-meta-auth-token')
if candidate_token:
path = quote('/v1/%s/.token%s/%s' %
path = quote('/v1/%s/.token_%s/%s' %
(self.auth_account, candidate_token[-1], candidate_token))
resp = self.make_request(req.environ, 'GET',
path).get_response(self.app)
@ -987,7 +995,7 @@ class Swauth(object):
# Generate new token
token = '%stk%s' % (self.reseller_prefix, uuid4().hex)
# Save token info
path = quote('/v1/%s/.token%s/%s' %
path = quote('/v1/%s/.token_%s/%s' %
(self.auth_account, token[-1], token))
resp = self.make_request(req.environ, 'PUT', path,
json.dumps({'account': account, 'user': user,
@ -1050,7 +1058,7 @@ class Swauth(object):
if expires < time():
groups = None
if not groups:
path = quote('/v1/%s/.token%s/%s' %
path = quote('/v1/%s/.token_%s/%s' %
(self.auth_account, token[-1], token))
resp = self.make_request(req.environ, 'GET',
path).get_response(self.app)
@ -1129,7 +1137,8 @@ class Swauth(object):
raise Exception(
'No memcache set up; required for Swauth middleware')
memcache_client.set(memcache_key, (self.itoken_expires,
'.auth,.reseller_admin'), timeout=self.token_life)
'.auth,.reseller_admin,%s.auth' % self.reseller_prefix),
timeout=self.token_life)
return self.itoken
def get_admin_detail(self, req):

2
test/unit/common/middleware/test_swauth.py
View File

@ -2882,7 +2882,7 @@ class TestAuth(unittest.TestCase):
self.assert_(itk.startswith('AUTH_itk'), itk)
expires, groups = fmc.get('AUTH_/auth/%s' % itk)
self.assert_(expires > time(), expires)
self.assertEquals(groups, '.auth,.reseller_admin')
self.assertEquals(groups, '.auth,.reseller_admin,AUTH_.auth')
def test_get_admin_detail_fail_no_colon(self):
self.test_auth.app = FakeApp(iter([]))