diff --git a/bin/swauth-add-account b/bin/swauth-add-account index fe18b5a72d..9f40429d00 100755 --- a/bin/swauth-add-account +++ b/bin/swauth-add-account @@ -51,11 +51,12 @@ if __name__ == '__main__': if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/%s' % (parsed.path, account) + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/%s' % (parsed_path, account) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} if options.suffix: diff --git a/bin/swauth-add-user b/bin/swauth-add-user index 045dc0a766..365a24a085 100755 --- a/bin/swauth-add-user +++ b/bin/swauth-add-user @@ -61,12 +61,13 @@ if __name__ == '__main__': if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' # Ensure the account exists - path = '%sv2/%s' % (parsed.path, account) + path = '%sv2/%s' % (parsed_path, account) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} if options.suffix: @@ -77,7 +78,7 @@ if __name__ == '__main__': if resp.status // 100 != 2: print 'Account creation failed: %s %s' % (resp.status, resp.reason) # Add the user - path = '%sv2/%s/%s' % (parsed.path, account, user) + path = '%sv2/%s/%s' % (parsed_path, account, user) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key, 'X-Auth-User-Key': password} diff --git a/bin/swauth-delete-account b/bin/swauth-delete-account index 3d98f6ec4e..8e4bd42f76 100755 --- a/bin/swauth-delete-account +++ b/bin/swauth-delete-account @@ -45,11 +45,12 @@ if __name__ == '__main__': if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/%s' % (parsed.path, account) + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/%s' % (parsed_path, account) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} conn = http_connect(parsed.hostname, parsed.port, 'DELETE', path, headers, diff --git a/bin/swauth-delete-user b/bin/swauth-delete-user index ede076dd5b..f2f7808b64 100755 --- a/bin/swauth-delete-user +++ b/bin/swauth-delete-user @@ -45,11 +45,12 @@ if __name__ == '__main__': if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/%s/%s' % (parsed.path, account, user) + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/%s/%s' % (parsed_path, account, user) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} conn = http_connect(parsed.hostname, parsed.port, 'DELETE', path, headers, diff --git a/bin/swauth-list b/bin/swauth-list index 85a7633966..217b5baff8 100755 --- a/bin/swauth-list +++ b/bin/swauth-list @@ -64,11 +64,12 @@ If the [user] is '.groups', the active groups for the account will be listed. if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/%s' % (parsed.path, '/'.join(args)) + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/%s' % (parsed_path, '/'.join(args)) headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} conn = http_connect(parsed.hostname, parsed.port, 'GET', path, headers, diff --git a/bin/swauth-prep b/bin/swauth-prep index 3d2cb7d3eb..ca47ff9faa 100755 --- a/bin/swauth-prep +++ b/bin/swauth-prep @@ -44,11 +44,12 @@ if __name__ == '__main__': if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/.prep' % parsed.path + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/.prep' % parsed_path headers = {'X-Auth-Admin-User': options.admin_user, 'X-Auth-Admin-Key': options.admin_key} conn = http_connect(parsed.hostname, parsed.port, 'POST', path, headers, diff --git a/bin/swauth-set-account-service b/bin/swauth-set-account-service index 054e4cfc4b..538210e497 100755 --- a/bin/swauth-set-account-service +++ b/bin/swauth-set-account-service @@ -55,11 +55,12 @@ Example: %prog -K swauthkey test storage local http://127.0.0.1:8080/v1/AUTH_018 if parsed.scheme not in ('http', 'https'): raise Exception('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(options.admin_url))) - if not parsed.path: - parsed.path = '/' - elif parsed.path[-1] != '/': - parsed.path += '/' - path = '%sv2/%s/.services' % (parsed.path, account) + parsed_path = parsed.path + if not parsed_path: + parsed_path = '/' + elif parsed_path[-1] != '/': + parsed_path += '/' + path = '%sv2/%s/.services' % (parsed_path, account) body = json.dumps({service: {name: url}}) headers = {'Content-Length': str(len(body)), 'X-Auth-Admin-User': options.admin_user, diff --git a/bin/swift-auth-add-user b/bin/swift-auth-add-user deleted file mode 100755 index 6b997d8ccd..0000000000 --- a/bin/swift-auth-add-user +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ConfigParser import ConfigParser -from optparse import OptionParser -from os.path import basename -from sys import argv, exit - -from swift.common.bufferedhttp import http_connect_raw as http_connect - - -if __name__ == '__main__': - default_conf = '/etc/swift/auth-server.conf' - parser = OptionParser( - usage='Usage: %prog [options] ') - parser.add_option('-c', '--conf', dest='conf', default=default_conf, - help='Configuration file to determine how to connect to the local ' - 'auth server (default: %s).' % default_conf) - parser.add_option('-a', '--admin', dest='admin', action='store_true', - default=False, help='Give the user administrator access; otherwise ' - 'the user will only have access to containers specifically allowed ' - 'with ACLs.') - parser.add_option('-r', '--reseller-admin', dest='reseller_admin', - action='store_true', default=False, help='Give the user full reseller ' - 'administrator access, giving them full access to all accounts within ' - 'the reseller, including the ability to create new accounts. Creating ' - 'a new reseller admin requires super_admin rights.') - parser.add_option('-U', '--admin-user', dest='admin_user', - default='.super_admin', help='The user with admin rights to add users ' - '(default: .super_admin).') - parser.add_option('-K', '--admin-key', dest='admin_key', - help='The key for the user with admin rights to add users.') - args = argv[1:] - if not args: - args.append('-h') - (options, args) = parser.parse_args(args) - if len(args) != 3: - parser.parse_args(['-h']) - account, user, password = args - c = ConfigParser() - if not c.read(options.conf): - exit('Unable to read conf file: %s' % options.conf) - conf = dict(c.items('app:auth-server')) - host = conf.get('bind_ip', '127.0.0.1') - port = int(conf.get('bind_port', 11000)) - ssl = conf.get('cert_file') is not None - path = '/account/%s/%s' % (account, user) - headers = {'X-Auth-Admin-User': options.admin_user, - 'X-Auth-Admin-Key': options.admin_key, - 'X-Auth-User-Key': password} - if options.admin: - headers['X-Auth-User-Admin'] = 'true' - if options.reseller_admin: - headers['X-Auth-User-Reseller-Admin'] = 'true' - conn = http_connect(host, port, 'PUT', path, headers, ssl=ssl) - resp = conn.getresponse() - if resp.status == 204: - print resp.getheader('x-storage-url') - else: - print 'Update failed: %s %s' % (resp.status, resp.reason) diff --git a/bin/swift-auth-recreate-accounts b/bin/swift-auth-recreate-accounts deleted file mode 100755 index 430940f44e..0000000000 --- a/bin/swift-auth-recreate-accounts +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ConfigParser import ConfigParser -from optparse import OptionParser -from sys import argv, exit - -from swift.common.bufferedhttp import http_connect_raw as http_connect - -if __name__ == '__main__': - default_conf = '/etc/swift/auth-server.conf' - parser = OptionParser(usage='Usage: %prog [options]') - parser.add_option('-c', '--conf', dest='conf', default=default_conf, - help='Configuration file to determine how to connect to the local ' - 'auth server (default: %s).' % default_conf) - parser.add_option('-U', '--admin-user', dest='admin_user', - default='.super_admin', help='The user with admin rights to recreate ' - 'accounts (default: .super_admin).') - parser.add_option('-K', '--admin-key', dest='admin_key', - help='The key for the user with admin rights to recreate accounts.') - args = argv[1:] - if not args: - args.append('-h') - (options, args) = parser.parse_args(args) - c = ConfigParser() - if not c.read(options.conf): - exit('Unable to read conf file: %s' % options.conf) - conf = dict(c.items('app:auth-server')) - host = conf.get('bind_ip', '127.0.0.1') - port = int(conf.get('bind_port', 11000)) - ssl = conf.get('cert_file') is not None - path = '/recreate_accounts' - conn = http_connect(host, port, 'POST', path, ssl=ssl, - headers={'X-Auth-Admin-User': options.admin_user, - 'X-Auth-Admin-Key': options.admin_key}) - resp = conn.getresponse() - if resp.status == 200: - print resp.read() - else: - print 'Recreating accounts failed. (%d)' % resp.status diff --git a/bin/swift-auth-server b/bin/swift-auth-server deleted file mode 100755 index 10c0fba073..0000000000 --- a/bin/swift-auth-server +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from swift.common.utils import parse_options -from swift.common.wsgi import run_wsgi - -if __name__ == '__main__': - conf_file, options = parse_options() - run_wsgi(conf_file, 'auth-server', default_port=11000, **options) diff --git a/bin/swift-auth-to-swauth b/bin/swift-auth-to-swauth deleted file mode 100755 index e1010c315a..0000000000 --- a/bin/swift-auth-to-swauth +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import gettext -from subprocess import call -from sys import argv, exit - -import sqlite3 - - -if __name__ == '__main__': - gettext.install('swift', unicode=1) - if len(argv) != 2: - exit('Syntax: %s ' % argv[0]) - _junk, auth_db = argv - conn = sqlite3.connect(auth_db) - try: - listing = conn.execute('SELECT account, cfaccount, user, password, ' - 'admin, reseller_admin FROM account') - except sqlite3.OperationalError, err: - listing = conn.execute('SELECT account, cfaccount, user, password, ' - '"f", "f" FROM account') - for account, cfaccount, user, password, admin, reseller_admin in listing: - cmd = ['swauth-add-user', '-K', '', '-s', - cfaccount.split('_', 1)[1]] - if admin == 't': - cmd.append('-a') - if reseller_admin == 't': - cmd.append('-r') - cmd.extend([account, user, password]) - print ' '.join(cmd) diff --git a/bin/swift-auth-update-reseller-prefixes b/bin/swift-auth-update-reseller-prefixes deleted file mode 100755 index 52b6345e99..0000000000 --- a/bin/swift-auth-update-reseller-prefixes +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/python -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os.path import basename -from sys import argv, exit - -from swift.common.db import get_db_connection - - -if __name__ == '__main__': - app = basename(argv[0]) - if len(argv) != 3: - exit(''' -Syntax : %s -Example: %s /etc/swift/auth.db AUTH'''.strip() % (app, app)) - db = argv[1] - new_prefix = argv[2].rstrip('_') - print 'Updating %s' % db - conn = get_db_connection(db) - rows = conn.execute('SELECT url, cfaccount FROM account').fetchall() - for row in rows: - old_prefix = '' - uuid = row[1] - if '_' in row[1]: - old_prefix, uuid = row[1].split('_', 1) - new_cfaccount = '%s_%s' % (new_prefix, uuid) - new_url = row[0].replace(row[1], new_cfaccount) - print '%s ->\n%s' % (row[0], new_url) - print '%s ->\n%s' % (row[1], new_cfaccount) - print - conn.execute('''UPDATE account SET url = ?, cfaccount = ? - WHERE url = ? AND cfaccount = ?''', - (new_url, new_cfaccount, row[0], row[1])) - conn.commit() - print 'Updated %s rows.' % len(rows) diff --git a/doc/source/_static/tweaks.css b/doc/source/_static/tweaks.css index 16cd6e76e2..1b6fdaa186 100644 --- a/doc/source/_static/tweaks.css +++ b/doc/source/_static/tweaks.css @@ -62,4 +62,151 @@ table.docutils { a tt { color:#CF2F19; -} \ No newline at end of file +} + +/* ------------------------------------------ +PURE CSS SPEECH BUBBLES +by Nicolas Gallagher +- http://nicolasgallagher.com/pure-css-speech-bubbles/ + +http://nicolasgallagher.com +http://twitter.com/necolas + +Created: 02 March 2010 +Version: 1.1 (21 October 2010) + +Dual licensed under MIT and GNU GPLv2 © Nicolas Gallagher +------------------------------------------ */ +/* THE SPEECH BUBBLE +------------------------------------------------------------------------------------------------------------------------------- */ + +/* THE SPEECH BUBBLE +------------------------------------------------------------------------------------------------------------------------------- */ + +.triangle-border { + position:relative; + padding:15px; + margin:1em 0 3em; + border:5px solid #BC1518; + color:#333; + background:#fff; + + /* css3 */ + -moz-border-radius:10px; + -webkit-border-radius:10px; + border-radius:10px; +} + +/* Variant : for left positioned triangle +------------------------------------------ */ + +.triangle-border.left { + margin-left:30px; +} + +/* Variant : for right positioned triangle +------------------------------------------ */ + +.triangle-border.right { + margin-right:30px; +} + +/* THE TRIANGLE +------------------------------------------------------------------------------------------------------------------------------- */ + +.triangle-border:before { + content:""; + display:block; /* reduce the damage in FF3.0 */ + position:absolute; + bottom:-40px; /* value = - border-top-width - border-bottom-width */ + left:40px; /* controls horizontal position */ + width:0; + height:0; + border:20px solid transparent; + border-top-color:#BC1518; +} + +/* creates the smaller triangle */ +.triangle-border:after { + content:""; + display:block; /* reduce the damage in FF3.0 */ + position:absolute; + bottom:-26px; /* value = - border-top-width - border-bottom-width */ + left:47px; /* value = (:before left) + (:before border-left) - (:after border-left) */ + width:0; + height:0; + border:13px solid transparent; + border-top-color:#fff; +} + +/* Variant : top +------------------------------------------ */ + +/* creates the larger triangle */ +.triangle-border.top:before { + top:-40px; /* value = - border-top-width - border-bottom-width */ + right:40px; /* controls horizontal position */ + bottom:auto; + left:auto; + border:20px solid transparent; + border-bottom-color:#BC1518; +} + +/* creates the smaller triangle */ +.triangle-border.top:after { + top:-26px; /* value = - border-top-width - border-bottom-width */ + right:47px; /* value = (:before right) + (:before border-right) - (:after border-right) */ + bottom:auto; + left:auto; + border:13px solid transparent; + border-bottom-color:#fff; +} + +/* Variant : left +------------------------------------------ */ + +/* creates the larger triangle */ +.triangle-border.left:before { + top:10px; /* controls vertical position */ + left:-30px; /* value = - border-left-width - border-right-width */ + bottom:auto; + border-width:15px 30px 15px 0; + border-style:solid; + border-color:transparent #BC1518; +} + +/* creates the smaller triangle */ +.triangle-border.left:after { + top:16px; /* value = (:before top) + (:before border-top) - (:after border-top) */ + left:-21px; /* value = - border-left-width - border-right-width */ + bottom:auto; + border-width:9px 21px 9px 0; + border-style:solid; + border-color:transparent #fff; +} + +/* Variant : right +------------------------------------------ */ + +/* creates the larger triangle */ +.triangle-border.right:before { + top:10px; /* controls vertical position */ + right:-30px; /* value = - border-left-width - border-right-width */ + bottom:auto; + left:auto; + border-width:15px 0 15px 30px; + border-style:solid; + border-color:transparent #BC1518; +} + +/* creates the smaller triangle */ +.triangle-border.right:after { + top:16px; /* value = (:before top) + (:before border-top) - (:after border-top) */ + right:-21px; /* value = - border-left-width - border-right-width */ + bottom:auto; + left:auto; + border-width:9px 0 9px 21px; + border-style:solid; + border-color:transparent #fff; +} + diff --git a/doc/source/_theme/layout.html b/doc/source/_theme/layout.html index ed1cab0a6e..f0573a3b43 100644 --- a/doc/source/_theme/layout.html +++ b/doc/source/_theme/layout.html @@ -1,2 +1,69 @@ {% extends "sphinxdoc/layout.html" %} {% set css_files = css_files + ['_static/tweaks.css'] %} + +{%- macro sidebar() %} +{%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+
+ {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

{{ _('Table Of Contents') }}

+ {{ toc }} + {%- endif %} + {%- endblock %} + {%- block sidebarrel %} + {%- if prev %} +

{{ _('Previous topic') }}

+

{{ prev.title }}

+ {%- endif %} + {%- if next %} +

{{ _('Next topic') }}

+

{{ next.title }}

+ {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

{{ _('This Page') }}

+ + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + +

+ Psst... hey. Did you know you can read Swift 1.2 docs or Swift 1.1 docs also? +

+ {%- endif %} + + {%- endblock %} +
+
+ {%- endif %}{% endif %} +{%- endmacro %} \ No newline at end of file diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index 05580a143e..bb3eef6fa6 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -159,15 +159,12 @@ of the cluster, we need to run the swift-stats-report tool to check the health of each of these containers and objects. These tools need direct access to the entire cluster and to the ring files -(installing them on an auth server or a proxy server will probably do). Both +(installing them on a proxy server will probably do). Both swift-stats-populate and swift-stats-report use the same configuration file, /etc/swift/stats.conf. Example conf file:: [stats] - # For DevAuth: - auth_url = http://saio:11000/v1.0 - # For Swauth: - # auth_url = http://saio:11000/auth/v1.0 + auth_url = http://saio:11000/auth/v1.0 auth_user = test:tester auth_key = testing @@ -236,15 +233,16 @@ then be graphed to see how cluster performance is trending. Additional Cleanup Script for Swauth ------------------------------------ -If you decide to use Swauth, you'll want to install a cronjob to clean up any +With Swauth, you'll want to install a cronjob to clean up any orphaned expired tokens. These orphaned tokens can occur when a "stampede" occurs where a single user authenticates several times concurrently. Generally, these orphaned tokens don't pose much of an issue, but it's good to clean them up once a "token life" period (default: 1 day or 86400 seconds). -This should be as simple as adding `swauth-cleanup-tokens -K swauthkey > -/dev/null` to a crontab entry on one of the proxies that is running Swauth; but -run `swauth-cleanup-tokens` with no arguments for detailed help on the options +This should be as simple as adding `swauth-cleanup-tokens -A +https://:8080/auth/ -K swauthkey > /dev/null` to a crontab +entry on one of the proxies that is running Swauth; but run +`swauth-cleanup-tokens` with no arguments for detailed help on the options available. ------------------------ diff --git a/doc/source/auth.rst b/doc/source/auth.rst deleted file mode 100644 index feb3be8a99..0000000000 --- a/doc/source/auth.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _auth: - -************************* -Developer's Authorization -************************* - -.. _auth_server: - -Auth Server -=========== - -.. automodule:: swift.auth.server - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/source/conf.py b/doc/source/conf.py index 88bc171e66..d03d956b9a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -60,7 +60,7 @@ master_doc = 'index' # General information about the project. project = u'Swift' -copyright = u'2010, OpenStack, LLC' +copyright = u'2011, OpenStack, LLC' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -220,5 +220,6 @@ latex_documents = [ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'python': ('http://docs.python.org/', None), - 'nova': ('http://nova.openstack.org', None)} + 'nova': ('http://nova.openstack.org', None), + 'glance': ('http://glance.openstack.org', None)} diff --git a/doc/source/development_auth.rst b/doc/source/development_auth.rst index 0f28750bd3..e0f3fcc7ba 100644 --- a/doc/source/development_auth.rst +++ b/doc/source/development_auth.rst @@ -6,13 +6,11 @@ Auth Server and Middleware Creating Your Own Auth Server and Middleware -------------------------------------------- -The included swift/auth/server.py and swift/common/middleware/auth.py are good -minimal examples of how to create an external auth server and proxy server auth -middleware. Also, see swift/common/middleware/swauth.py for -a more complete implementation. The main points are that the auth middleware -can reject requests up front, before they ever get to the Swift Proxy -application, and afterwards when the proxy issues callbacks to verify -authorization. +The included swift/common/middleware/swauth.py is a good example of how to +create an auth subsystem with proxy server auth middleware. The main points are +that the auth middleware can reject requests up front, before they ever get to +the Swift Proxy application, and afterwards when the proxy issues callbacks to +verify authorization. It's generally good to separate the authentication and authorization procedures. Authentication verifies that a request actually comes from who it @@ -29,7 +27,7 @@ specific information, it just passes it along. Convention has environ['REMOTE_USER'] set to the authenticated user string but often more information is needed than just that. -The included DevAuth will set the REMOTE_USER to a comma separated list of +The included Swauth will set the REMOTE_USER to a comma separated list of groups the user belongs to. The first group will be the "user's group", a group that only the user belongs to. The second group will be the "account's group", a group that includes all users for that auth account (different than the @@ -39,7 +37,7 @@ will be omitted. It is highly recommended that authentication server implementers prefix their tokens and Swift storage accounts they create with a configurable reseller -prefix (`AUTH_` by default with the included DevAuth). This prefix will avoid +prefix (`AUTH_` by default with the included Swauth). This prefix will avoid conflicts with other authentication servers that might be using the same Swift cluster. Otherwise, the Swift cluster will have to try all the resellers until one validates a token or all fail. @@ -48,22 +46,20 @@ A restriction with group names is that no group name should begin with a period '.' as that is reserved for internal Swift use (such as the .r for referrer designations as you'll see later). -Example Authentication with DevAuth: +Example Authentication with Swauth: - * Token AUTH_tkabcd is given to the DevAuth middleware in a request's + * Token AUTH_tkabcd is given to the Swauth middleware in a request's X-Auth-Token header. - * The DevAuth middleware makes a validate token AUTH_tkabcd call to the - external DevAuth server. - * The external DevAuth server validates the token AUTH_tkabcd and discovers + * The Swauth middleware validates the token AUTH_tkabcd and discovers it matches the "tester" user within the "test" account for the storage account "AUTH_storage_xyz". - * The external DevAuth server responds with "X-Auth-Groups: - test:tester,test,AUTH_storage_xyz" + * The Swauth server sets the REMOTE_USER to + "test:tester,test,AUTH_storage_xyz" * Now this user will have full access (via authorization procedures later) to the AUTH_storage_xyz Swift storage account and access to containers in other storage accounts, provided the storage account begins with the same `AUTH_` reseller prefix and the container has an ACL specifying at least - one of those three groups returned. + one of those three groups. Authorization is performed through callbacks by the Swift Proxy server to the WSGI environment's swift.authorize value, if one is set. The swift.authorize @@ -283,11 +279,9 @@ sometimes that's less important than meeting certain ACL requirements. Integrating With repoze.what ---------------------------- -Here's an example of integration with repoze.what, though honestly it just does -what the default swift/common/middleware/auth.py does in a slightly different -way. I'm no repoze.what expert by any stretch; this is just included here to -hopefully give folks a start on their own code if they want to use -repoze.what:: +Here's an example of integration with repoze.what, though honestly I'm no +repoze.what expert by any stretch; this is just included here to hopefully give +folks a start on their own code if they want to use repoze.what:: from time import time diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index f7963b1b24..9d0bce0a19 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -182,6 +182,46 @@ Setting up rsync #. `service rsync restart` +--------------------------------------------------- +Optional: Setting up rsyslog for individual logging +--------------------------------------------------- + + #. Create /etc/rsyslog.d/10-swift.conf:: + + # Uncomment the following to have a log containing all logs together + #local1,local2,local3,local4,local5.* /var/log/swift/all.log + + # Uncomment the following to have hourly proxy logs for stats processing + #$template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%" + #local1.*;local1.!notice ?HourlyProxyLog + + local1.*;local1.!notice /var/log/swift/proxy.log + local1.notice /var/log/swift/proxy.error + local1.* ~ + + local2.*;local2.!notice /var/log/swift/storage1.log + local2.notice /var/log/swift/storage1.error + local2.* ~ + + local3.*;local3.!notice /var/log/swift/storage2.log + local3.notice /var/log/swift/storage2.error + local3.* ~ + + local4.*;local4.!notice /var/log/swift/storage3.log + local4.notice /var/log/swift/storage3.error + local4.* ~ + + local5.*;local5.!notice /var/log/swift/storage4.log + local5.notice /var/log/swift/storage4.error + local5.* ~ + + #. Edit /etc/rsyslog.conf and make the following change:: + + $PrivDropToGroup adm + + #. `mkdir -p /var/log/swift/hourly` + #. `chown -R syslog.adm /var/log/swift` + #. `service rsyslog restart` ------------------------------------------------ Getting the code and setting up test environment @@ -215,43 +255,20 @@ Configuring each node Sample configuration files are provided with all defaults in line-by-line comments. - #. If you're going to use the DevAuth (the default swift-auth-server), create - `/etc/swift/auth-server.conf` (you can skip this if you're going to use - Swauth):: - - [DEFAULT] - user = - - [pipeline:main] - pipeline = auth-server - - [app:auth-server] - use = egg:swift#auth - default_cluster_url = http://127.0.0.1:8080/v1 - # Highly recommended to change this. - super_admin_key = devauth - #. Create `/etc/swift/proxy-server.conf`:: [DEFAULT] bind_port = 8080 user = + log_facility = LOG_LOCAL1 [pipeline:main] - # For DevAuth: - pipeline = healthcheck cache auth proxy-server - # For Swauth: - # pipeline = healthcheck cache swauth proxy-server + pipeline = healthcheck cache swauth proxy-server [app:proxy-server] use = egg:swift#proxy allow_account_management = true - # Only needed for DevAuth - [filter:auth] - use = egg:swift#auth - - # Only needed for Swauth [filter:swauth] use = egg:swift#swauth # Highly recommended to change this. @@ -276,6 +293,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6012 user = + log_facility = LOG_LOCAL2 [pipeline:main] pipeline = account-server @@ -297,6 +315,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6022 user = + log_facility = LOG_LOCAL3 [pipeline:main] pipeline = account-server @@ -318,6 +337,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6032 user = + log_facility = LOG_LOCAL4 [pipeline:main] pipeline = account-server @@ -339,6 +359,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6042 user = + log_facility = LOG_LOCAL5 [pipeline:main] pipeline = account-server @@ -360,6 +381,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6011 user = + log_facility = LOG_LOCAL2 [pipeline:main] pipeline = container-server @@ -381,6 +403,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6021 user = + log_facility = LOG_LOCAL3 [pipeline:main] pipeline = container-server @@ -402,6 +425,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6031 user = + log_facility = LOG_LOCAL4 [pipeline:main] pipeline = container-server @@ -423,6 +447,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6041 user = + log_facility = LOG_LOCAL5 [pipeline:main] pipeline = container-server @@ -445,6 +470,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6010 user = + log_facility = LOG_LOCAL2 [pipeline:main] pipeline = object-server @@ -466,6 +492,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6020 user = + log_facility = LOG_LOCAL3 [pipeline:main] pipeline = object-server @@ -487,6 +514,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6030 user = + log_facility = LOG_LOCAL4 [pipeline:main] pipeline = object-server @@ -508,6 +536,7 @@ Sample configuration files are provided with all defaults in line-by-line commen mount_check = false bind_port = 6040 user = + log_facility = LOG_LOCAL5 [pipeline:main] pipeline = object-server @@ -531,6 +560,7 @@ Setting up scripts for running Swift #!/bin/bash swift-init all stop + find /var/log/swift -type f -exec rm -f {} \; sudo umount /mnt/sdb1 sudo mkfs.xfs -f -i size=1024 /dev/sdb1 sudo mount /mnt/sdb1 @@ -573,14 +603,12 @@ Setting up scripts for running Swift #!/bin/bash swift-init main start - # The auth-server line is only needed for DevAuth: - swift-init auth-server start - #. For Swauth (not needed for DevAuth), create `~/bin/recreateaccounts`:: + #. Create `~/bin/recreateaccounts`:: #!/bin/bash - # Replace devauth with whatever your super_admin key is (recorded in + # Replace swauthkey with whatever your super_admin key is (recorded in # /etc/swift/proxy-server.conf). swauth-prep -K swauthkey swauth-add-user -K swauthkey -a test tester testing @@ -592,24 +620,17 @@ Setting up scripts for running Swift #!/bin/bash - # Replace devauth with whatever your super_admin key is (recorded in - # /etc/swift/auth-server.conf). This swift-auth-recreate-accounts line - # is only needed for DevAuth: - swift-auth-recreate-accounts -K devauth swift-init rest start #. `chmod +x ~/bin/*` #. `remakerings` #. `cd ~/swift/trunk; ./.unittests` #. `startmain` (The ``Unable to increase file descriptor limit. Running as non-root?`` warnings are expected and ok.) - #. For Swauth: `recreateaccounts` - #. For DevAuth: `swift-auth-add-user -K devauth -a test tester testing` # Replace ``devauth`` with whatever your super_admin key is (recorded in /etc/swift/auth-server.conf). - #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:11000/v1.0`` # For Swauth, make the last URL `http://127.0.0.1:8080/auth/v1.0` + #. `recreateaccounts` + #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` - #. Check that `st` works: `st -A http://127.0.0.1:11000/v1.0 -U test:tester -K testing stat` # For Swauth, make the URL `http://127.0.0.1:8080/auth/v1.0` - #. For DevAuth: `swift-auth-add-user -K devauth -a test2 tester2 testing2` # Replace ``devauth`` with whatever your super_admin key is (recorded in /etc/swift/auth-server.conf). - #. For DevAuth: `swift-auth-add-user -K devauth test tester3 testing3` # Replace ``devauth`` with whatever your super_admin key is (recorded in /etc/swift/auth-server.conf). - #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` # For Swauth, add auth_prefix = /auth/ and change auth_port = 8080. + #. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` + #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete everything in the configured accounts.) #. `cd ~/swift/trunk; ./.probetests` (Note: probe tests will reset your @@ -634,7 +655,7 @@ If all doesn't go as planned, and tests fail, or you can't auth, or something do #. Everything is logged in /var/log/syslog, so that is a good first place to look for errors (most likely python tracebacks). #. Make sure all of the server processes are running. For the base - functionality, the Proxy, Account, Container, Object and Auth servers + functionality, the Proxy, Account, Container, and Object servers should be running #. If one of the servers are not running, and no errors are logged to syslog, it may be useful to try to start the server manually, for example: diff --git a/doc/source/howto_cyberduck.rst b/doc/source/howto_cyberduck.rst deleted file mode 100644 index e9de135ff3..0000000000 --- a/doc/source/howto_cyberduck.rst +++ /dev/null @@ -1,160 +0,0 @@ -=============================== -Talking to Swift with Cyberduck -=============================== - -.. note:: - Put together by Caleb Tennis, thanks Caleb! - - -#. Install Swift, or have credentials for an existing Swift installation. If - you plan to install Swift on your own server, follow the general guidelines - in the section following this one. (This documentation assumes the use of - the DevAuth auth server; if you're using Swauth, you should change all auth - URLs /v1.0 to /auth/v1.0) - -#. Verify you can connect using the standard Swift Tool `st` from your - "public" URL (yes I know this resolves privately inside EC2):: - - ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ st -A https://ec2-184-72-156-130.compute-1.amazonaws.com:11000/v1.0 -U a3:b3 -K c3 stat - Account: 06228ccf-6d0a-4395-889e-e971e8de8781 - Containers: 0 - Objects: 0 - Bytes: 0 - - .. note:: - - The Swift Tool `st` can be copied from Swift sources to most any - machine with Python installed. You can grab it from - http://bazaar.launchpad.net/%7Ehudson-openstack/swift/trunk/annotate/head%3A/bin/st - if you don't have the Swift code handy. - -#. Download and extract the Cyberduck sources (3.5.1 as of this writing). They - should be available at http://trac.cyberduck.ch/ - -#. Edit the Cyberduck source. Look for lib/cloudfiles.properties, and edit - this file. Change auth_url to your public auth URL (note the https):: - - auth_url=https://ec2-184-72-156-130.compute-1.amazonaws.com:11000/v1.0 - -#. Edit source/ch/cyberduck/core/Protocol.java. Look for the line saying - "storage.clouddrive.com". Just above that, change:: - - public boolean isHostnameConfigurable() { - return true; - } - -#. In the root directory, run "make" to rebuild Cyberduck. When done, type: - `open build/Release/Cyberduck.app/` to start the program. - -#. Go to "Open Connection", select Rackspace Cloud Files, and connect. - - .. image:: howto_cyberduck_config.png - -#. If you get SSL errors, make sure your auth and proxy server are both setup - for SSL. If you get certificate errors (specifically, 'unable to find valid - certification path to requested target'), you are using a self signed - certificate, you need to perform a few more steps: - - .. note:: - - For some folks, just telling the OS to trust the cert works fine, for - others use the following steps. - -#. As outlined here: http://blogs.sun.com/andreas/entry/no_more_unable_to_find, - download http://blogs.sun.com/andreas/resource/InstallCert.java, run "javac - InstallCert.java" to compile it, then run "java InstallCert - https://your-auth-server-url:8080". This script will pull down that - certificate and put it into a Java cert store, in your local directory. The - file is jssecacerts. - -#. You need to move that file to $JAVA_HOME/jre/lib/security, so your java run - time picks it up. - -#. Restart Cyberduck, and it should now allow you to use that certificate - without an error. - - ---------------------------------------- -Installing Swift For Use With Cyberduck ---------------------------------------- - -#. Both the proxy and auth servers will ultimately need to be running with - SSL. You will need a key and certificate to do this, self signed is ok (but - a little more work getting Cyberduck to accept it). Put these in - /etc/swift/cert.crt and /etc/swift/cert.key. - - .. note:: - - Creating a self-signed cert can usually be done with:: - - cd /etc/swift - openssl req -new -x509 -nodes -out cert.crt -keyout cert.key - -#. Example proxy-server config:: - - [DEFAULT] - cert_file = /etc/swift/cert.crt - key_file = /etc/swift/cert.key - - [pipeline:main] - pipeline = healthcheck cache auth proxy-server - - [app:proxy-server] - use = egg:swift#proxy - - [filter:auth] - use = egg:swift#auth - ssl = true - - [filter:healthcheck] - use = egg:swift#healthcheck - - [filter:cache] - use = egg:swift#memcache - -#. Example auth-server config:: - - [DEFAULT] - cert_file = /etc/swift/cert.crt - key_file = /etc/swift/cert.key - - [pipeline:main] - pipeline = auth-server - - [app:auth-server] - use = egg:swift#auth - super_admin_key = devauth - default_cluster_url = https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1 - -#. Use swift-auth-add-user to create a new account and admin user:: - - ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ swift-auth-add-user -K devauth -a a3 b3 c3 - https://ec2-184-72-156-130.compute-1.amazonaws.com:8080/v1/06228ccf-6d0a-4395-889e-e971e8de8781 - - .. note:: - It's important that the URL that is given back to you be accessible - publicly. This URL is tied to this account, and will be served - back to Cyberduck after authorization. If this URL gives back - something like: http://127.0.0.1/v1/... this won't work, because - Cyberduck will attempt to connect to 127.0.0.1. - - This URL is specified in the auth-server config's - default_cluster_url. However, once you have created an - account/user, this URL is fixed and won't change even if you change - that configuration item. You will have to use sqlite to manually - edit the auth.db in order to change it (limitation of using the - development auth server, but perhaps someone will patch in this - ability someday). - -#. Verify you can connect using the standard Swift Tool `st`:: - - ubuntu@domU-12-31-39-03-CD-06:/home/swift/swift/bin$ st -A https://127.0.0.1:11000/v1.0 -U a3:b3 -K c3 stat - Account: 06228ccf-6d0a-4395-889e-e971e8de8781 - Containers: 0 - Objects: 0 - Bytes: 0 - -.. note:: - - Please let me know if you find any changes that need to be made: ctennis on - IRC diff --git a/doc/source/howto_cyberduck_config.png b/doc/source/howto_cyberduck_config.png deleted file mode 100644 index 1612bbe0e6..0000000000 Binary files a/doc/source/howto_cyberduck_config.png and /dev/null differ diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index a314474993..ddc1ddd044 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -13,8 +13,7 @@ Prerequisites Basic architecture and terms ---------------------------- - *node* - a host machine running one or more Swift services -- *Proxy node* - node that runs Proxy services; can also run Swauth -- *Auth node* - node that runs the Auth service; only required for DevAuth +- *Proxy node* - node that runs Proxy services; also runs Swauth - *Storage node* - node that runs Account, Container, and Object services - *ring* - a set of mappings of Swift data to physical devices @@ -23,15 +22,9 @@ This document shows a cluster using the following types of nodes: - one Proxy node - Runs the swift-proxy-server processes which proxy requests to the - appropriate Storage nodes. For Swauth, the proxy server will also contain + appropriate Storage nodes. The proxy server will also contain the Swauth service as WSGI middleware. -- one Auth node - - - Runs the swift-auth-server which controls authentication and - authorization for all requests. This can be on the same node as a - Proxy node. This is only required for DevAuth. - - five Storage nodes - Runs the swift-account-server, swift-container-server, and @@ -92,7 +85,6 @@ General OS configuration and partitioning for each node export STORAGE_LOCAL_NET_IP=10.1.2.3 export PROXY_LOCAL_NET_IP=10.1.2.4 - export AUTH_LOCAL_NET_IP=10.1.2.5 .. note:: The random string of text in /etc/swift/swift.conf is @@ -136,26 +128,14 @@ Configure the Proxy node bind_port = 8080 workers = 8 user = swift - # For non-local Auth server - ip = $AUTH_LOCAL_NET_IP - [pipeline:main] - # For DevAuth: - pipeline = healthcheck cache auth proxy-server - # For Swauth: - # pipeline = healthcheck cache swauth proxy-server + pipeline = healthcheck cache swauth proxy-server [app:proxy-server] use = egg:swift#proxy allow_account_management = true - # Only needed for DevAuth - [filter:auth] - use = egg:swift#auth - ssl = true - - # Only needed for Swauth [filter:swauth] use = egg:swift#swauth default_swift_cluster = local#https://$PROXY_LOCAL_NET_IP:8080/v1 @@ -228,42 +208,6 @@ Configure the Proxy node swift-init proxy start -Configure the Auth node ------------------------ - -.. note:: Only required for DevAuth; you can skip this section for Swauth. - -#. If this node is not running on the same node as a proxy, create a - self-signed cert as you did for the Proxy node - -#. Install swift-auth service:: - - apt-get install swift-auth - -#. Create /etc/swift/auth-server.conf:: - - cat >/etc/swift/auth-server.conf <:8080/v1 - # Highly recommended to change this key to something else! - super_admin_key = devauth - EOF - -#. Start Auth services:: - - swift-init auth start - chown swift:swift /etc/swift/auth.db - swift-init auth restart # 1.1.0 workaround because swift creates auth.db owned as root - Configure the Storage nodes --------------------------- @@ -418,26 +362,21 @@ replicator, updater, or auditor.:: Create Swift admin account and test ----------------------------------- -You run these commands from the Auth node. - -.. note:: For Swauth, replace the https://:11000/v1.0 with - https://:8080/auth/v1.0 +You run these commands from the Proxy node. #. Create a user with administrative privileges (account = system, username = root, password = testpass). Make sure to replace - ``devauth`` (or ``swauthkey``) with whatever super_admin key you assigned in - the auth-server.conf file (or proxy-server.conf file in the case of Swauth) + ``swauthkey`` with whatever super_admin key you assigned in + the proxy-server.conf file above. *Note: None of the values of account, username, or password are special - they can be anything.*:: - # For DevAuth: - swift-auth-add-user -K devauth -a system root testpass - # For Swauth: - swauth-add-user -K swauthkey -a system root testpass + swauth-prep -A https://:8080/auth/ -K swauthkey + swauth-add-user -A https://:8080/auth/ -K swauthkey -a system root testpass #. Get an X-Storage-Url and X-Auth-Token:: - curl -k -v -H 'X-Storage-User: system:root' -H 'X-Storage-Pass: testpass' https://:11000/v1.0 + curl -k -v -H 'X-Storage-User: system:root' -H 'X-Storage-Pass: testpass' https://:8080/auth/v1.0 #. Check that you can HEAD the account:: @@ -445,32 +384,32 @@ You run these commands from the Auth node. #. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes):: - st -A https://:11000/v1.0 -U system:root -K testpass stat + st -A https://:8080/auth/v1.0 -U system:root -K testpass stat #. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: - st -A https://:11000/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz - st -A https://:11000/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz + st -A https://:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz + st -A https://:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz #. Use ``st`` to download all files from the 'myfiles' container:: - st -A https://:11000/v1.0 -U system:root -K testpass download myfiles + st -A https://:8080/auth/v1.0 -U system:root -K testpass download myfiles #. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: - st -A https://:11000/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder + st -A https://:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder #. Use ``st`` to list your containers:: - st -A https://:11000/v1.0 -U system:root -K testpass list + st -A https://:8080/auth/v1.0 -U system:root -K testpass list #. Use ``st`` to list the contents of your 'builders' container:: - st -A https://:11000/v1.0 -U system:root -K testpass list builders + st -A https://:8080/auth/v1.0 -U system:root -K testpass list builders #. Use ``st`` to download all files from the 'builders' container:: - st -A https://:11000/v1.0 -U system:root -K testpass download builders + st -A https://:8080/auth/v1.0 -U system:root -K testpass download builders .. _add-proxy-server: @@ -489,31 +428,25 @@ See :ref:`config-proxy` for the initial setup, and then follow these additional use = egg:swift#memcache memcache_servers = :11211 -#. Change the default_cluster_url to point to the load balanced url, rather than the first proxy server you created in /etc/swift/auth-server.conf (for DevAuth) or in /etc/swift/proxy-server.conf (for Swauth):: +#. Change the default_cluster_url to point to the load balanced url, rather than the first proxy server you created in /etc/swift/proxy-server.conf:: - # For DevAuth, in /etc/swift/auth-server.conf - [app:auth-server] - use = egg:swift#auth - default_cluster_url = https:///v1 - # Highly recommended to change this key to something else! - super_admin_key = devauth - - # For Swauth, in /etc/swift/proxy-server.conf [filter:swauth] use = egg:swift#swauth default_swift_cluster = local#http:///v1 # Highly recommended to change this key to something else! super_admin_key = swauthkey -#. For DevAuth, after you change the default_cluster_url setting, you have to delete the auth database and recreate the Swift users, or manually update the auth database with the correct URL for each account. +#. The above will make new accounts with the new default_swift_cluster URL, however it won't change any existing accounts. You can change a service URL for existing accounts with:: - For Swauth, you can change a service URL with:: + First retreve what the URL was:: - swauth-set-account-service -K swauthkey storage local + swauth-list -A https://:8080/auth/ -K swauthkey - You can obtain old service URLs with:: - - swauth-list -K swauthkey + And then update it with:: + + swauth-set-account-service -A https://:8080/auth/ -K swauthkey storage local + + Make the look just like it's original URL but with the host:port update you want. #. Next, copy all the ring information to all the nodes, including your new proxy nodes, and ensure the ring info gets to all the storage nodes as well. @@ -522,15 +455,16 @@ See :ref:`config-proxy` for the initial setup, and then follow these additional Additional Cleanup Script for Swauth ------------------------------------ -If you decide to use Swauth, you'll want to install a cronjob to clean up any +With Swauth, you'll want to install a cronjob to clean up any orphaned expired tokens. These orphaned tokens can occur when a "stampede" occurs where a single user authenticates several times concurrently. Generally, these orphaned tokens don't pose much of an issue, but it's good to clean them up once a "token life" period (default: 1 day or 86400 seconds). -This should be as simple as adding `swauth-cleanup-tokens -K swauthkey > -/dev/null` to a crontab entry on one of the proxies that is running Swauth; but -run `swauth-cleanup-tokens` with no arguments for detailed help on the options +This should be as simple as adding `swauth-cleanup-tokens -A +https://:8080/auth/ -K swauthkey > /dev/null` to a crontab +entry on one of the proxies that is running Swauth; but run +`swauth-cleanup-tokens` with no arguments for detailed help on the options available. Troubleshooting Notes diff --git a/doc/source/index.rst b/doc/source/index.rst index 3c23140e72..3091dfdf49 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -67,14 +67,6 @@ Administrator Documentation admin_guide debian_package_guide -End User Guides -=============== - -.. toctree:: - :maxdepth: 1 - - howto_cyberduck - Source Documentation ==================== @@ -87,7 +79,6 @@ Source Documentation container db object - auth misc diff --git a/doc/source/misc.rst b/doc/source/misc.rst index db77e464f3..5fae5f9a4c 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -33,15 +33,6 @@ Utils :members: :show-inheritance: -.. _common_auth: - -Auth -==== - -.. automodule:: swift.common.middleware.auth - :members: - :show-inheritance: - .. _common_swauth: Swauth diff --git a/doc/source/overview_auth.rst b/doc/source/overview_auth.rst index f9141509c2..027b77dfba 100644 --- a/doc/source/overview_auth.rst +++ b/doc/source/overview_auth.rst @@ -2,61 +2,57 @@ The Auth System =============== --------------- -Developer Auth --------------- - -The auth system for Swift is loosely based on the auth system from the existing -Rackspace architecture -- actually from a few existing auth systems -- and is -therefore a bit disjointed. The distilled points about it are: - -* The authentication/authorization part is outside Swift itself -* The user of Swift passes in an auth token with each request -* Swift validates each token with the external auth system and caches the - result -* The token does not change from request to request, but does expire - -The token can be passed into Swift using the X-Auth-Token or the -X-Storage-Token header. Both have the same format: just a simple string -representing the token. Some external systems use UUID tokens, some an MD5 hash -of something unique, some use "something else" but the salient point is that -the token is a string which can be sent as-is back to the auth system for -validation. - -Swift will make calls to the external auth system, giving the auth token to be -validated. For a valid token, the auth system responds with an overall -expiration in seconds from now. Swift will cache the token up to the expiration -time. The included devauth also has the concept of admin and non-admin users -within an account. Admin users can do anything within the account. Non-admin -users can only perform operations per container based on the container's -X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see -:mod:`swift.common.middleware.acl` - -The user starts a session by sending a ReST request to the external auth system -to receive the auth token and a URL to the Swift system. - --------------- -Extending Auth --------------- - -Auth is written as wsgi middleware, so implementing your own auth is as easy -as writing new wsgi middleware, and plugging it in to the proxy server. - -The current middleware is implemented in the DevAuthMiddleware class in -swift/common/middleware/auth.py, and should be a good starting place for -implementing your own auth. - -Also, see :doc:`development_auth`. - - ------ Swauth ------ -The Swauth system is an optional DevAuth replacement included at -swift/common/middleware/swauth.py; a scalable authentication and -authorization system that uses Swift itself as its backing store. This section -will describe how it stores its data. +The auth system for Swift is loosely based on the auth system from the existing +Rackspace architecture -- actually from a few existing auth systems -- and is +therefore a bit disjointed. The distilled points about it are: + +* The authentication/authorization part can be an external system or a + subsystem run within Swift as WSGI middleware +* The user of Swift passes in an auth token with each request +* Swift validates each token with the external auth system or auth subsystem + and caches the result +* The token does not change from request to request, but does expire + +The token can be passed into Swift using the X-Auth-Token or the +X-Storage-Token header. Both have the same format: just a simple string +representing the token. Some auth systems use UUID tokens, some an MD5 hash of +something unique, some use "something else" but the salient point is that the +token is a string which can be sent as-is back to the auth system for +validation. + +Swift will make calls to the auth system, giving the auth token to be +validated. For a valid token, the auth system responds with an overall +expiration in seconds from now. Swift will cache the token up to the expiration +time. The included Swauth also has the concept of admin and non-admin users +within an account. Admin users can do anything within the account. Non-admin +users can only perform operations per container based on the container's +X-Container-Read and X-Container-Write ACLs. For more information on ACLs, see +:mod:`swift.common.middleware.acl` + +The user starts a session by sending a ReST request to the auth system to +receive the auth token and a URL to the Swift system. + +-------------- +Extending Auth +-------------- + +Swauth is written as wsgi middleware, so implementing your own auth is as easy +as writing new wsgi middleware, and plugging it in to the proxy server. + +Also, see :doc:`development_auth`. + + +-------------- +Swauth Details +-------------- + +The Swauth system is included at swift/common/middleware/swauth.py; a scalable +authentication and authorization system that uses Swift itself as its backing +store. This section will describe how it stores its data. At the topmost level, the auth system has its own Swift account it stores its own account information within. This Swift account is known as diff --git a/doc/source/overview_stats.rst b/doc/source/overview_stats.rst index 111e1f8df0..07632f6d43 100644 --- a/doc/source/overview_stats.rst +++ b/doc/source/overview_stats.rst @@ -89,45 +89,28 @@ Running the stats system on SAIO #. Create a swift account to use for storing stats information, and note the account hash. The hash will be used in config files. -#. Install syslog-ng:: +#. Edit /etc/rsyslog.d/10-swift.conf:: - sudo apt-get install syslog-ng + # Uncomment the following to have a log containing all logs together + #local1,local2,local3,local4,local5.* /var/log/swift/all.log -#. Add the following to the end of `/etc/syslog-ng/syslog-ng.conf`:: + $template HourlyProxyLog,"/var/log/swift/hourly/%$YEAR%%$MONTH%%$DAY%%$HOUR%" + local1.*;local1.!notice ?HourlyProxyLog - # Added for swift logging - destination df_local1 { file("/var/log/swift/proxy.log" owner() group()); }; - destination df_local1_err { file("/var/log/swift/proxy.error" owner() group()); }; - destination df_local1_hourly { file("/var/log/swift/hourly/$YEAR$MONTH$DAY$HOUR" owner() group()); }; - filter f_local1 { facility(local1) and level(info); }; + local1.*;local1.!notice /var/log/swift/proxy.log + local1.notice /var/log/swift/proxy.error + local1.* ~ - filter f_local1_err { facility(local1) and not level(info); }; - - # local1.info -/var/log/swift/proxy.log - # write to local file and to remove log server - log { - source(s_all); - filter(f_local1); - destination(df_local1); - destination(df_local1_hourly); - }; - - # local1.error -/var/log/swift/proxy.error - # write to local file and to remove log server - log { - source(s_all); - filter(f_local1_err); - destination(df_local1_err); - }; - -#. Restart syslog-ng - -#. Create the log directories:: - - mkdir /var/log/swift/hourly - mkdir /var/log/swift/stats - chown -R : /var/log/swift +#. Edit /etc/rsyslog.conf and make the following change:: + + $PrivDropToGroup adm +#. `mkdir -p /var/log/swift/hourly` +#. `chown -R syslog.adm /var/log/swift` +#. `chmod 775 /var/log/swift /var/log/swift/hourly` +#. `service rsyslog restart` +#. `usermod -a -G adm ` +#. Relogin to let the group change take effect. #. Create `/etc/swift/log-processor.conf`:: [log-processor] diff --git a/etc/auth-server.conf-sample b/etc/auth-server.conf-sample deleted file mode 100644 index 711f48d564..0000000000 --- a/etc/auth-server.conf-sample +++ /dev/null @@ -1,30 +0,0 @@ -# Only needed for DevAuth; Swauth is within the proxy-server.conf -[DEFAULT] -# bind_ip = 0.0.0.0 -# bind_port = 11000 -# workers = 1 -# user = swift -# swift_dir = /etc/swift -# cert_file = Default is no cert; format is path like /etc/swift/auth.crt -# key_file = Default is no key; format is path like /etc/swift/auth.key -# You can specify default log routing here if you want: -# log_name = swift -# log_facility = LOG_LOCAL0 -# log_level = INFO - -[pipeline:main] -pipeline = auth-server - -[app:auth-server] -use = egg:swift#auth -# Highly recommended to change this. -super_admin_key = devauth -# You can override the default log routing for this app here: -# set log_name = proxy-server -# set log_facility = LOG_LOCAL0 -# set log_level = INFO -# set log_headers = False -# reseller_prefix = AUTH -# default_cluster_url = http://127.0.0.1:8080/v1 -# token_life = 86400 -# node_timeout = 10 diff --git a/etc/log-processor.conf-sample b/etc/log-processor.conf-sample index 7619d0599a..4e33e70c48 100644 --- a/etc/log-processor.conf-sample +++ b/etc/log-processor.conf-sample @@ -14,7 +14,7 @@ swift_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 # log_dir = /var/log/swift/ swift_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 container_name = log_data -source_filename_format = access-%Y%m%d%H +source_filename_pattern = access-%Y%m%d%H # new_log_cutoff = 7200 # unlink_log = True class_path = swift.stats.access_processor.AccessLogProcessor @@ -31,9 +31,9 @@ class_path = swift.stats.access_processor.AccessLogProcessor # log_dir = /var/log/swift/ swift_account = AUTH_7abbc116-8a07-4b63-819d-02715d3e0f31 container_name = account_stats -source_filename_format = stats-%Y%m%d%H_* +source_filename_pattern = stats-%Y%m%d%H_.* # new_log_cutoff = 7200 # unlink_log = True class_path = swift.stats.stats_processor.StatsLogProcessor # account_server_conf = /etc/swift/account-server.conf -# user = swift \ No newline at end of file +# user = swift diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 3af7db0f8a..457aa42cf3 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -13,10 +13,7 @@ # log_level = INFO [pipeline:main] -# For DevAuth: -pipeline = catch_errors healthcheck cache ratelimit auth proxy-server -# For Swauth: -# pipeline = catch_errors healthcheck cache ratelimit swauth proxy-server +pipeline = catch_errors healthcheck cache ratelimit swauth proxy-server [app:proxy-server] use = egg:swift#proxy @@ -44,27 +41,6 @@ use = egg:swift#proxy # 'false' no one, even authorized, can. # allow_account_management = false -# Only needed for DevAuth -[filter:auth] -use = egg:swift#auth -# You can override the default log routing for this filter here: -# set log_name = auth-server -# set log_facility = LOG_LOCAL0 -# set log_level = INFO -# set log_headers = False -# The reseller prefix will verify a token begins with this prefix before even -# attempting to validate it with the external authentication server. Also, with -# authorization, only Swift storage accounts with this prefix will be -# authorized by this middleware. Useful if multiple auth systems are in use for -# one Swift cluster. -# reseller_prefix = AUTH -# ip = 127.0.0.1 -# port = 11000 -# ssl = false -# prefix = / -# node_timeout = 10 - -# Only needed for Swauth [filter:swauth] use = egg:swift#swauth # You can override the default log routing for this filter here: @@ -97,7 +73,7 @@ super_admin_key = swauthkey [filter:healthcheck] use = egg:swift#healthcheck # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = healthcheck # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False @@ -105,7 +81,7 @@ use = egg:swift#healthcheck [filter:cache] use = egg:swift#memcache # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = cache # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False @@ -116,7 +92,7 @@ use = egg:swift#memcache [filter:ratelimit] use = egg:swift#ratelimit # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = ratelimit # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False @@ -148,7 +124,7 @@ use = egg:swift#ratelimit [filter:domain_remap] use = egg:swift#domain_remap # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = domain_remap # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False @@ -159,7 +135,7 @@ use = egg:swift#domain_remap [filter:catch_errors] use = egg:swift#catch_errors # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = catch_errors # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False @@ -168,7 +144,7 @@ use = egg:swift#catch_errors # Note: this middleware requires python-dnspython use = egg:swift#cname_lookup # You can override the default log routing for this filter here: -# set log_name = auth-server +# set log_name = cname_lookup # set log_facility = LOG_LOCAL0 # set log_level = INFO # set log_headers = False diff --git a/etc/stats.conf-sample b/etc/stats.conf-sample index f89cb77d6d..e9bceadb1f 100644 --- a/etc/stats.conf-sample +++ b/etc/stats.conf-sample @@ -1,8 +1,5 @@ [stats] -# For DevAuth: -auth_url = http://saio:11000/auth -# For Swauth: -# auth_url = http://saio:8080/auth/v1.0 +auth_url = http://saio:8080/auth/v1.0 auth_user = test:tester auth_key = testing # swift_dir = /etc/swift diff --git a/setup.py b/setup.py index c80d62ddc8..868ecde88a 100644 --- a/setup.py +++ b/setup.py @@ -79,9 +79,6 @@ setup( 'bin/st', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/swift-account-replicator', 'bin/swift-account-server', - 'bin/swift-auth-add-user', - 'bin/swift-auth-recreate-accounts', 'bin/swift-auth-server', - 'bin/swift-auth-update-reseller-prefixes', 'bin/swift-container-auditor', 'bin/swift-container-replicator', 'bin/swift-container-server', 'bin/swift-container-updater', @@ -100,7 +97,7 @@ setup( 'bin/swauth-add-account', 'bin/swauth-add-user', 'bin/swauth-cleanup-tokens', 'bin/swauth-delete-account', 'bin/swauth-delete-user', 'bin/swauth-list', 'bin/swauth-prep', - 'bin/swauth-set-account-service', 'bin/swift-auth-to-swauth', + 'bin/swauth-set-account-service', ], entry_points={ 'paste.app_factory': [ @@ -108,10 +105,8 @@ setup( 'object=swift.obj.server:app_factory', 'container=swift.container.server:app_factory', 'account=swift.account.server:app_factory', - 'auth=swift.auth.server:app_factory', ], 'paste.filter_factory': [ - 'auth=swift.common.middleware.auth:filter_factory', 'swauth=swift.common.middleware.swauth:filter_factory', 'healthcheck=swift.common.middleware.healthcheck:filter_factory', 'memcache=swift.common.middleware.memcache:filter_factory', diff --git a/swift/account/server.py b/swift/account/server.py index f15ac38c11..54f13177f5 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -261,7 +261,7 @@ class AccountController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) try: - args = simplejson.load(req.body_file) + args = simplejson.load(req.environ['wsgi.input']) except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) diff --git a/swift/auth/__init__.py b/swift/auth/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/swift/auth/server.py b/swift/auth/server.py deleted file mode 100644 index 4f5ae4b21c..0000000000 --- a/swift/auth/server.py +++ /dev/null @@ -1,693 +0,0 @@ -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import with_statement -import os -import sys -from contextlib import contextmanager -from time import gmtime, strftime, time -from urllib import unquote, quote -from uuid import uuid4 -from hashlib import md5, sha1 -import hmac -import base64 - -import sqlite3 -from webob import Request, Response -from webob.exc import HTTPBadRequest, HTTPConflict, HTTPForbidden, \ - HTTPNoContent, HTTPUnauthorized, HTTPServiceUnavailable, HTTPNotFound - -from swift.common.bufferedhttp import http_connect_raw as http_connect -from swift.common.db import get_db_connection -from swift.common.utils import get_logger, split_path, urlparse - - -class AuthController(object): - """ - Sample implementation of an authorization server for development work. This - server only implements the basic functionality and isn't written for high - availability or to scale to thousands (or even hundreds) of requests per - second. It is mainly for use by developers working on the rest of the - system. - - The design of the auth system was restricted by a couple of existing - systems. - - This implementation stores an account name, user name, and password (in - plain text!) as well as a corresponding Swift cluster url and account hash. - One existing auth system used account, user, and password whereas another - used just account and an "API key". Here, we support both systems with - their various, sometimes colliding headers. - - The most common use case is by the end user: - - * The user makes a ReST call to the auth server requesting a token and url - to use to access the Swift cluster. - * The auth system validates the user info and returns a token and url for - the user to use with the Swift cluster. - * The user makes a ReST call to the Swift cluster using the url given with - the token as the X-Auth-Token header. - * The Swift cluster makes an ReST call to the auth server to validate the - token, caching the result for future requests up to the expiration the - auth server returns. - * The auth server validates the token given and returns the expiration for - the token. - * The Swift cluster completes the user's request. - - Another use case is creating a new user: - - * The developer makes a ReST call to create a new user. - * If the account for the user does not yet exist, the auth server makes - a ReST call to the Swift cluster to create a new account on its end. - * The auth server records the information in its database. - - A last use case is recreating existing accounts; this is really only useful - on a development system when the drives are reformatted quite often but - the auth server's database is retained: - - * A developer makes an ReST call to have the existing accounts recreated. - * For each account in its database, the auth server makes a ReST call to - the Swift cluster to create the specific account on its end. - - :param conf: The [auth-server] dictionary of the auth server configuration - file - - See the etc/auth-server.conf-sample for information on the possible - configuration parameters. - """ - - def __init__(self, conf): - self.logger = get_logger(conf, log_route='auth-server') - self.super_admin_key = conf.get('super_admin_key') - if not self.super_admin_key: - msg = _('No super_admin_key set in conf file! Exiting.') - try: - self.logger.critical(msg) - except Exception: - pass - raise ValueError(msg) - self.swift_dir = conf.get('swift_dir', '/etc/swift') - self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() - if self.reseller_prefix and self.reseller_prefix[-1] != '_': - self.reseller_prefix += '_' - self.default_cluster_url = conf.get('default_cluster_url', - 'http://127.0.0.1:8080/v1').rstrip('/') - self.token_life = int(conf.get('token_life', 86400)) - self.log_headers = conf.get('log_headers') == 'True' - self.db_file = os.path.join(self.swift_dir, 'auth.db') - self.conn = get_db_connection(self.db_file, okay_to_create=True) - try: - self.conn.execute('SELECT admin FROM account LIMIT 1') - except sqlite3.OperationalError, err: - if str(err) == 'no such column: admin': - self.conn.execute("ALTER TABLE account ADD COLUMN admin TEXT") - self.conn.execute("UPDATE account SET admin = 't'") - try: - self.conn.execute('SELECT reseller_admin FROM account LIMIT 1') - except sqlite3.OperationalError, err: - if str(err) == 'no such column: reseller_admin': - self.conn.execute( - "ALTER TABLE account ADD COLUMN reseller_admin TEXT") - self.conn.execute('''CREATE TABLE IF NOT EXISTS account ( - account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT, admin TEXT, - reseller_admin TEXT)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account - ON account (account)''') - try: - self.conn.execute('SELECT user FROM token LIMIT 1') - except sqlite3.OperationalError, err: - if str(err) == 'no such column: user': - self.conn.execute('DROP INDEX IF EXISTS ix_token_created') - self.conn.execute('DROP INDEX IF EXISTS ix_token_cfaccount') - self.conn.execute('DROP TABLE IF EXISTS token') - self.conn.execute('''CREATE TABLE IF NOT EXISTS token ( - token TEXT, created FLOAT, - account TEXT, user TEXT, cfaccount TEXT)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token - ON token (token)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created - ON token (created)''') - self.conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account - ON token (account)''') - self.conn.commit() - for row in self.conn.execute('SELECT cfaccount FROM account'): - if not row[0].startswith(self.reseller_prefix): - previous_prefix = '' - if '_' in row[0]: - previous_prefix = row[0].split('_', 1)[0] - msg = (_(''' -THERE ARE ACCOUNTS IN YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW RESELLER -PREFIX OF "%(reseller)s". -YOU HAVE A FEW OPTIONS: - 1. RUN "swift-auth-update-reseller-prefixes %(db_file)s %(reseller)s", - "swift-init auth-server restart", AND - "swift-auth-recreate-accounts -K ..." TO CREATE FRESH ACCOUNTS. - OR - 2. REMOVE %(db_file)s, RUN "swift-init auth-server restart", AND RUN - "swift-auth-add-user ..." TO CREATE BRAND NEW ACCOUNTS THAT WAY. - OR - 3. ADD "reseller_prefix = %(previous)s" (WITHOUT THE QUOTES) TO YOUR - proxy-server.conf IN THE [filter:auth] SECTION AND TO YOUR - auth-server.conf IN THE [app:auth-server] SECTION AND RUN - "swift-init proxy-server restart" AND "swift-init auth-server restart" - TO REVERT BACK TO YOUR PREVIOUS RESELLER PREFIX. - - %(note)s - ''') % {'reseller': self.reseller_prefix.rstrip('_'), - 'db_file': self.db_file, - 'previous': previous_prefix, - 'note': previous_prefix and ' ' or _(''' - SINCE YOUR PREVIOUS RESELLER PREFIX WAS AN EMPTY STRING, IT IS NOT - RECOMMENDED TO PERFORM OPTION 3 AS THAT WOULD MAKE SUPPORTING MULTIPLE - RESELLERS MORE DIFFICULT. - ''').strip()}).strip() - self.logger.critical(_('CRITICAL: ') + ' '.join(msg.split())) - raise Exception('\n' + msg) - - def add_storage_account(self, account_name=''): - """ - Creates an account within the Swift cluster by making a ReST call. - - :param account_name: The desired name for the account; if omitted a - UUID4 will be used. - :returns: False upon failure, otherwise the name of the account - within the Swift cluster. - """ - orig_account_name = account_name - if not account_name: - account_name = '%s%s' % (self.reseller_prefix, uuid4().hex) - url = '%s/%s' % (self.default_cluster_url, account_name) - parsed = urlparse(url) - # Create a single use token. - token = '%stk%s' % (self.reseller_prefix, uuid4().hex) - with self.get_conn() as conn: - conn.execute(''' - INSERT INTO token - (token, created, account, user, cfaccount) VALUES - (?, ?, '.super_admin', '.single_use', '.reseller_admin')''', - (token, time())) - conn.commit() - if parsed.port is None: - port = {'http': 80, 'https': 443}.get(parsed.scheme, 80) - else: - port = parsed.port - conn = http_connect(parsed.hostname, port, 'PUT', parsed.path, - {'X-Auth-Token': token}, ssl=(parsed.scheme == 'https')) - resp = conn.getresponse() - resp.read() - if resp.status // 100 != 2: - self.logger.error(_('ERROR attempting to create account %(url)s:' \ - ' %(status)s %(reason)s') % - {'url': url, 'status': resp.status, 'reason': resp.reason}) - return False - return account_name - - @contextmanager - def get_conn(self): - """ - Returns a DB API connection instance to the auth server's SQLite - database. This is a contextmanager call to be use with the 'with' - statement. It takes no parameters. - """ - if not self.conn: - # We go ahead and make another db connection even if this is a - # reentry call; just in case we had an error that caused self.conn - # to become None. Even if we make an extra conn, we'll only keep - # one after the 'with' block. - self.conn = get_db_connection(self.db_file) - conn = self.conn - self.conn = None - try: - yield conn - conn.rollback() - self.conn = conn - except Exception, err: - try: - conn.close() - except Exception: - pass - self.conn = get_db_connection(self.db_file) - raise err - - def validate_s3_sign(self, request, token): - account, user, sign = \ - request.headers['Authorization'].split(' ')[-1].split(':') - msg = base64.urlsafe_b64decode(unquote(token)) - rv = False - with self.get_conn() as conn: - row = conn.execute(''' - SELECT password, cfaccount FROM account - WHERE account = ? AND user = ?''', - (account, user)).fetchone() - rv = (84000, account, user, row[1]) - if rv: - s = base64.encodestring(hmac.new(row[0], msg, - sha1).digest()).strip() - self.logger.info("orig %s, calc %s" % (sign, s)) - if sign != s: - rv = False - return rv - - def purge_old_tokens(self): - """ - Removes tokens that have expired from the auth server's database. This - is called by :func:`validate_token` and :func:`GET` to help keep the - database clean. - """ - with self.get_conn() as conn: - conn.execute('DELETE FROM token WHERE created < ?', - (time() - self.token_life,)) - conn.commit() - - def validate_token(self, token): - """ - Tests if the given token is a valid token - - :param token: The token to validate - :returns: (TTL, account, user, cfaccount) if valid, False otherwise. - cfaccount will be None for users without admin access for the - account. cfaccount will be .reseller_admin for users with - full reseller admin rights. - """ - begin = time() - self.purge_old_tokens() - rv = False - with self.get_conn() as conn: - row = conn.execute(''' - SELECT created, account, user, cfaccount FROM token - WHERE token = ?''', - (token,)).fetchone() - if row is not None: - created = row[0] - if time() - created < self.token_life: - rv = (self.token_life - (time() - created), row[1], row[2], - row[3]) - # Remove the token if it was expired or single use. - if not rv or rv[2] == '.single_use': - conn.execute(''' - DELETE FROM token WHERE token = ?''', (token,)) - conn.commit() - self.logger.info('validate_token(%s, _, _) = %s [%.02f]' % - (repr(token), repr(rv), time() - begin)) - return rv - - def create_user(self, account, user, password, admin=False, - reseller_admin=False): - """ - Handles the create_user call for developers, used to request a user be - added in the auth server database. If the account does not yet exist, - it will be created on the Swift cluster and the details recorded in the - auth server database. - - The url for the storage account is constructed now and stored - separately to support changing the configuration file's - default_cluster_url for directing new accounts to a different Swift - cluster while still supporting old accounts going to the Swift clusters - they were created on. - - Currently, updating a user's information (password, admin access) must - be done by directly updating the sqlite database. - - :param account: The name for the new account - :param user: The name for the new user - :param password: The password for the new account - :param admin: If true, the user will be granted full access to the - account; otherwise, another user will have to add the - user to the ACLs for containers to grant access. - :param reseller_admin: If true, the user will be granted full access to - all accounts within this reseller, including the - ability to create additional accounts. - - :returns: False if the create fails, 'already exists' if the user - already exists, or storage url if successful - """ - begin = time() - if not all((account, user, password)): - return False - with self.get_conn() as conn: - row = conn.execute( - 'SELECT url FROM account WHERE account = ? AND user = ?', - (account, user)).fetchone() - if row: - self.logger.info(_('ALREADY EXISTS create_user(%(account)s, ' - '%(user)s, _, %(admin)s, %(reseller_admin)s) ' - '[%(elapsed).02f]') % - {'account': repr(account), - 'user': repr(user), - 'admin': repr(admin), - 'reseller_admin': repr(reseller_admin), - 'elapsed': time() - begin}) - return 'already exists' - row = conn.execute( - 'SELECT url, cfaccount FROM account WHERE account = ?', - (account,)).fetchone() - if row: - url = row[0] - account_hash = row[1] - else: - account_hash = self.add_storage_account() - if not account_hash: - self.logger.info(_('FAILED create_user(%(account)s, ' - '%(user)s, _, %(admin)s, %(reseller_admin)s) ' - '[%(elapsed).02f]') % - {'account': repr(account), - 'user': repr(user), - 'admin': repr(admin), - 'reseller_admin': repr(reseller_admin), - 'elapsed': time() - begin}) - return False - url = self.default_cluster_url.rstrip('/') + '/' + account_hash - conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password, admin, - reseller_admin) - VALUES (?, ?, ?, ?, ?, ?, ?)''', - (account, url, account_hash, user, password, - admin and 't' or '', reseller_admin and 't' or '')) - conn.commit() - self.logger.info(_('SUCCESS create_user(%(account)s, %(user)s, _, ' - '%(admin)s, %(reseller_admin)s) = %(url)s [%(elapsed).02f]') % - {'account': repr(account), 'user': repr(user), - 'admin': repr(admin), 'reseller_admin': repr(reseller_admin), - 'url': repr(url), 'elapsed': time() - begin}) - return url - - def recreate_accounts(self): - """ - Recreates the accounts from the existing auth database in the Swift - cluster. This is useful on a development system when the drives are - reformatted quite often but the auth server's database is retained. - - :returns: A string indicating accounts and failures - """ - begin = time() - with self.get_conn() as conn: - account_hashes = [r[0] for r in conn.execute( - 'SELECT distinct(cfaccount) FROM account').fetchall()] - failures = [] - for i, account_hash in enumerate(account_hashes): - if not self.add_storage_account(account_hash): - failures.append(account_hash) - rv = '%d accounts, failures %s' % (len(account_hashes), repr(failures)) - self.logger.info('recreate_accounts(_, _) = %s [%.02f]' % - (rv, time() - begin)) - return rv - - def is_account_admin(self, request, for_account): - """ - Returns True if the request represents coming from .super_admin, a - .reseller_admin, or an admin for the account specified. - """ - if request.headers.get('X-Auth-Admin-User') == '.super_admin' and \ - request.headers.get('X-Auth-Admin-Key') == self.super_admin_key: - return True - try: - account, user = \ - request.headers.get('X-Auth-Admin-User').split(':', 1) - except (AttributeError, ValueError): - return False - with self.get_conn() as conn: - row = conn.execute(''' - SELECT reseller_admin, admin FROM account - WHERE account = ? AND user = ? AND password = ?''', - (account, user, - request.headers.get('X-Auth-Admin-Key'))).fetchone() - if row: - if row[0] == 't': - return True - if row[1] == 't' and account == for_account: - return True - return False - - def handle_token(self, request): - """ - Handles ReST requests from Swift to validate tokens - - Valid URL paths: - * GET /token/ - - If the HTTP request returns with a 204, then the token is valid, the - TTL of the token will be available in the X-Auth-Ttl header, and a - comma separated list of the "groups" the user belongs to will be in the - X-Auth-Groups header. - - :param request: webob.Request object - """ - try: - _junk, token = split_path(request.path, minsegs=2) - except ValueError: - return HTTPBadRequest() - # Retrieves (TTL, account, user, cfaccount) if valid, False otherwise - headers = {} - if 'Authorization' in request.headers: - validation = self.validate_s3_sign(request, token) - if validation: - headers['X-Auth-Account-Suffix'] = validation[3] - else: - validation = self.validate_token(token) - if not validation: - return HTTPNotFound() - groups = ['%s:%s' % (validation[1], validation[2]), validation[1]] - if validation[3]: - # admin access to a cfaccount or ".reseller_admin" to access to all - # accounts, including creating new ones. - groups.append(validation[3]) - headers['X-Auth-TTL'] = validation[0] - headers['X-Auth-Groups'] = ','.join(groups) - return HTTPNoContent(headers=headers) - - def handle_add_user(self, request): - """ - Handles Rest requests from developers to have a user added. If the - account specified doesn't exist, it will also be added. Currently, - updating a user's information (password, admin access) must be done by - directly updating the sqlite database. - - Valid URL paths: - * PUT /account// - create the account - - Valid headers: - * X-Auth-User-Key: - * X-Auth-User-Admin: - * X-Auth-User-Reseller-Admin: - - If the HTTP request returns with a 204, then the user was added, - and the storage url will be available in the X-Storage-Url header. - - :param request: webob.Request object - """ - try: - _junk, account_name, user_name = \ - split_path(request.path, minsegs=3) - except ValueError: - return HTTPBadRequest() - create_reseller_admin = \ - request.headers.get('x-auth-user-reseller-admin') == 'true' - if create_reseller_admin and ( - request.headers.get('X-Auth-Admin-User') != '.super_admin' or - request.headers.get('X-Auth-Admin-Key') != self.super_admin_key): - return HTTPUnauthorized(request=request) - create_account_admin = \ - request.headers.get('x-auth-user-admin') == 'true' - if create_account_admin and \ - not self.is_account_admin(request, account_name): - return HTTPForbidden(request=request) - if 'X-Auth-User-Key' not in request.headers: - return HTTPBadRequest(body='X-Auth-User-Key is required') - password = request.headers['x-auth-user-key'] - storage_url = self.create_user(account_name, user_name, password, - create_account_admin, create_reseller_admin) - if storage_url == 'already exists': - return HTTPConflict(body=storage_url) - if not storage_url: - return HTTPServiceUnavailable() - return HTTPNoContent(headers={'x-storage-url': storage_url}) - - def handle_account_recreate(self, request): - """ - Handles ReST requests from developers to have accounts in the Auth - system recreated in Swift. I know this is bad ReST style, but this - isn't production right? :) - - Valid URL paths: - * POST /recreate_accounts - - :param request: webob.Request object - """ - if request.headers.get('X-Auth-Admin-User') != '.super_admin' or \ - request.headers.get('X-Auth-Admin-Key') != self.super_admin_key: - return HTTPUnauthorized(request=request) - result = self.recreate_accounts() - return Response(result, 200, request=request) - - def handle_auth(self, request): - """ - Handles ReST requests from end users for a Swift cluster url and auth - token. This can handle all the various headers and formats that - existing auth systems used, so it's a bit of a chameleon. - - Valid URL paths: - * GET /v1//auth - * GET /auth - * GET /v1.0 - - Valid headers: - * X-Auth-User: : - * X-Auth-Key: - * X-Storage-User: [:] - The [:] is only optional here if the - /v1//auth path is used. - * X-Storage-Pass: - - The (currently) preferred method is to use /v1.0 path and the - X-Auth-User and X-Auth-Key headers. - - :param request: A webob.Request instance. - """ - try: - pathsegs = split_path(request.path, minsegs=1, maxsegs=3, - rest_with_last=True) - except ValueError: - return HTTPBadRequest() - if pathsegs[0] == 'v1' and pathsegs[2] == 'auth': - account = pathsegs[1] - user = request.headers.get('x-storage-user') - if not user: - user = request.headers.get('x-auth-user') - if not user or ':' not in user: - return HTTPUnauthorized() - account2, user = user.split(':', 1) - if account != account2: - return HTTPUnauthorized() - password = request.headers.get('x-storage-pass') - if not password: - password = request.headers.get('x-auth-key') - elif pathsegs[0] in ('auth', 'v1.0'): - user = request.headers.get('x-auth-user') - if not user: - user = request.headers.get('x-storage-user') - if not user or ':' not in user: - return HTTPUnauthorized() - account, user = user.split(':', 1) - password = request.headers.get('x-auth-key') - if not password: - password = request.headers.get('x-storage-pass') - else: - return HTTPBadRequest() - if not all((account, user, password)): - return HTTPUnauthorized() - self.purge_old_tokens() - with self.get_conn() as conn: - row = conn.execute(''' - SELECT cfaccount, url, admin, reseller_admin FROM account - WHERE account = ? AND user = ? AND password = ?''', - (account, user, password)).fetchone() - if row is None: - return HTTPUnauthorized() - cfaccount = row[0] - url = row[1] - admin = row[2] == 't' - reseller_admin = row[3] == 't' - row = conn.execute(''' - SELECT token FROM token WHERE account = ? AND user = ?''', - (account, user)).fetchone() - if row: - token = row[0] - else: - token = '%stk%s' % (self.reseller_prefix, uuid4().hex) - token_cfaccount = '' - if admin: - token_cfaccount = cfaccount - if reseller_admin: - token_cfaccount = '.reseller_admin' - conn.execute(''' - INSERT INTO token - (token, created, account, user, cfaccount) - VALUES (?, ?, ?, ?, ?)''', - (token, time(), account, user, token_cfaccount)) - conn.commit() - return HTTPNoContent(headers={'x-auth-token': token, - 'x-storage-token': token, - 'x-storage-url': url}) - - def handleREST(self, env, start_response): - """ - Handles routing of ReST requests. This handler also logs all requests. - - :param env: WSGI environment - :param start_response: WSGI start_response function - """ - req = Request(env) - logged_headers = None - if self.log_headers: - logged_headers = '\n'.join('%s: %s' % (k, v) - for k, v in req.headers.items()).replace('"', "#042") - start_time = time() - # Figure out how to handle the request - try: - if req.method == 'GET' and req.path.startswith('/v1') or \ - req.path.startswith('/auth'): - handler = self.handle_auth - elif req.method == 'GET' and req.path.startswith('/token/'): - handler = self.handle_token - elif req.method == 'PUT' and req.path.startswith('/account/'): - handler = self.handle_add_user - elif req.method == 'POST' and \ - req.path == '/recreate_accounts': - handler = self.handle_account_recreate - else: - return HTTPBadRequest(request=env)(env, start_response) - response = handler(req) - except Exception: - self.logger.exception( - _('ERROR Unhandled exception in ReST request')) - return HTTPServiceUnavailable(request=req)(env, start_response) - trans_time = '%.4f' % (time() - start_time) - if not response.content_length and response.app_iter and \ - hasattr(response.app_iter, '__len__'): - response.content_length = sum(map(len, response.app_iter)) - the_request = '%s %s' % (req.method, quote(unquote(req.path))) - if req.query_string: - the_request = the_request + '?' + req.query_string - the_request += ' ' + req.environ['SERVER_PROTOCOL'] - client = req.headers.get('x-cluster-client-ip') - if not client and 'x-forwarded-for' in req.headers: - client = req.headers['x-forwarded-for'].split(',')[0].strip() - if not client: - client = req.remote_addr - self.logger.info( - '%s - - [%s] "%s" %s %s "%s" "%s" - - - - - - - - - "-" "%s" ' - '"%s" %s' % ( - client, - strftime('%d/%b/%Y:%H:%M:%S +0000', gmtime()), - the_request, - response.status_int, - response.content_length or '-', - req.referer or '-', - req.user_agent or '-', - req.remote_addr, - logged_headers or '-', - trans_time)) - return response(env, start_response) - - def __call__(self, env, start_response): - """ Used by the eventlet.wsgi.server """ - return self.handleREST(env, start_response) - - -def app_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - return AuthController(conf) diff --git a/swift/common/client.py b/swift/common/client.py index 1fffaa493d..34874aa4ca 100644 --- a/swift/common/client.py +++ b/swift/common/client.py @@ -18,16 +18,15 @@ Cloud Files client library used internally """ import socket from cStringIO import StringIO -from httplib import HTTPException from re import compile, DOTALL from tokenize import generate_tokens, STRING, NAME, OP from urllib import quote as _quote, unquote from urlparse import urlparse, urlunparse try: - from eventlet.green.httplib import HTTPSConnection + from eventlet.green.httplib import HTTPException, HTTPSConnection except ImportError: - from httplib import HTTPSConnection + from httplib import HTTPException, HTTPSConnection try: from eventlet import sleep diff --git a/swift/common/middleware/auth.py b/swift/common/middleware/auth.py deleted file mode 100644 index a51788f7b7..0000000000 --- a/swift/common/middleware/auth.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from time import time - -from eventlet.timeout import Timeout -from webob.exc import HTTPForbidden, HTTPUnauthorized, HTTPNotFound - -from swift.common.bufferedhttp import http_connect_raw as http_connect -from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed -from swift.common.utils import cache_from_env, split_path, TRUE_VALUES - - -class DevAuth(object): - """Auth Middleware that uses the dev auth server.""" - - def __init__(self, app, conf): - self.app = app - self.conf = conf - self.reseller_prefix = conf.get('reseller_prefix', 'AUTH').strip() - if self.reseller_prefix and self.reseller_prefix[-1] != '_': - self.reseller_prefix += '_' - self.auth_host = conf.get('ip', '127.0.0.1') - self.auth_port = int(conf.get('port', 11000)) - self.ssl = conf.get('ssl', 'false').lower() in TRUE_VALUES - self.auth_prefix = conf.get('prefix', '/') - self.timeout = int(conf.get('node_timeout', 10)) - - def __call__(self, env, start_response): - """ - Accepts a standard WSGI application call, authenticating the request - and installing callback hooks for authorization and ACL header - validation. For an authenticated request, REMOTE_USER will be set to a - comma separated list of the user's groups. - - With a non-empty reseller prefix, acts as the definitive auth service - for just tokens and accounts that begin with that prefix, but will deny - requests outside this prefix if no other auth middleware overrides it. - - With an empty reseller prefix, acts as the definitive auth service only - for tokens that validate to a non-empty set of groups. For all other - requests, acts as the fallback auth service when no other auth - middleware overrides it. - """ - s3 = env.get('HTTP_AUTHORIZATION') - token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) - if s3 or (token and token.startswith(self.reseller_prefix)): - # Note: Empty reseller_prefix will match all tokens. - # Attempt to auth my token with my auth server - groups = self.get_groups(env, token, - memcache_client=cache_from_env(env)) - if groups: - env['REMOTE_USER'] = groups - user = groups and groups.split(',', 1)[0] or '' - # We know the proxy logs the token, so we augment it just a bit - # to also log the authenticated user. - env['HTTP_X_AUTH_TOKEN'] = '%s,%s' % (user, token) - env['swift.authorize'] = self.authorize - env['swift.clean_acl'] = clean_acl - else: - # Unauthorized token - if self.reseller_prefix: - # Because I know I'm the definitive auth for this token, I - # can deny it outright. - return HTTPUnauthorized()(env, start_response) - # Because I'm not certain if I'm the definitive auth for empty - # reseller_prefixed tokens, I won't overwrite swift.authorize. - elif 'swift.authorize' not in env: - env['swift.authorize'] = self.denied_response - else: - if self.reseller_prefix: - # With a non-empty reseller_prefix, I would like to be called - # back for anonymous access to accounts I know I'm the - # definitive auth for. - try: - version, rest = split_path(env.get('PATH_INFO', ''), - 1, 2, True) - except ValueError: - return HTTPNotFound()(env, start_response) - if rest and rest.startswith(self.reseller_prefix): - # Handle anonymous access to accounts I'm the definitive - # auth for. - env['swift.authorize'] = self.authorize - env['swift.clean_acl'] = clean_acl - # Not my token, not my account, I can't authorize this request, - # deny all is a good idea if not already set... - elif 'swift.authorize' not in env: - env['swift.authorize'] = self.denied_response - # Because I'm not certain if I'm the definitive auth for empty - # reseller_prefixed accounts, I won't overwrite swift.authorize. - elif 'swift.authorize' not in env: - env['swift.authorize'] = self.authorize - env['swift.clean_acl'] = clean_acl - return self.app(env, start_response) - - def get_groups(self, env, token, memcache_client=None): - """ - Get groups for the given token. - - If memcache_client is set, token credentials will be cached - appropriately. - - With a cache miss, or no memcache_client, the configurated external - authentication server will be queried for the group information. - - :param token: Token to validate and return a group string for. - :param memcache_client: Memcached client to use for caching token - credentials; None if no caching is desired. - :returns: None if the token is invalid or a string containing a comma - separated list of groups the authenticated user is a member - of. The first group in the list is also considered a unique - identifier for that user. - """ - groups = None - key = '%s/token/%s' % (self.reseller_prefix, token) - cached_auth_data = memcache_client and memcache_client.get(key) - if cached_auth_data: - start, expiration, groups = cached_auth_data - if time() - start > expiration: - groups = None - - headers = {} - if env.get('HTTP_AUTHORIZATION'): - groups = None - headers["Authorization"] = env.get('HTTP_AUTHORIZATION') - - if not groups: - with Timeout(self.timeout): - conn = http_connect(self.auth_host, self.auth_port, 'GET', - '%stoken/%s' % (self.auth_prefix, token), - headers, ssl=self.ssl) - - resp = conn.getresponse() - resp.read() - conn.close() - if resp.status // 100 != 2: - return None - expiration = float(resp.getheader('x-auth-ttl')) - groups = resp.getheader('x-auth-groups') - if memcache_client: - memcache_client.set(key, (time(), expiration, groups), - timeout=expiration) - - if env.get('HTTP_AUTHORIZATION'): - account, user, sign = \ - env['HTTP_AUTHORIZATION'].split(' ')[-1].split(':') - cfaccount = resp.getheader('x-auth-account-suffix') - path = env['PATH_INFO'] - env['PATH_INFO'] = \ - path.replace("%s:%s" % (account, user), cfaccount, 1) - - return groups - - def authorize(self, req): - """ - Returns None if the request is authorized to continue or a standard - WSGI response callable if not. - """ - try: - version, account, container, obj = split_path(req.path, 1, 4, True) - except ValueError: - return HTTPNotFound(request=req) - if not account or not account.startswith(self.reseller_prefix): - return self.denied_response(req) - user_groups = (req.remote_user or '').split(',') - if '.reseller_admin' in user_groups: - return None - if account in user_groups and \ - (req.method not in ('DELETE', 'PUT') or container): - # If the user is admin for the account and is not trying to do an - # account DELETE or PUT... - return None - referrers, groups = parse_acl(getattr(req, 'acl', None)) - if referrer_allowed(req.referer, referrers): - return None - if not req.remote_user: - return self.denied_response(req) - for user_group in user_groups: - if user_group in groups: - return None - return self.denied_response(req) - - def denied_response(self, req): - """ - Returns a standard WSGI response callable with the status of 403 or 401 - depending on whether the REMOTE_USER is set or not. - """ - if req.remote_user: - return HTTPForbidden(request=req) - else: - return HTTPUnauthorized(request=req) - - -def filter_factory(global_conf, **local_conf): - """Returns a WSGI filter app for use with paste.deploy.""" - conf = global_conf.copy() - conf.update(local_conf) - - def auth_filter(app): - return DevAuth(app, conf) - return auth_filter diff --git a/swift/common/middleware/swift3.py b/swift/common/middleware/swift3.py index 423b0123be..7c92314102 100644 --- a/swift/common/middleware/swift3.py +++ b/swift/common/middleware/swift3.py @@ -60,7 +60,7 @@ import base64 import errno import boto.utils from xml.sax.saxutils import escape as xml_escape -import cgi +import urlparse from webob import Request, Response from webob.exc import HTTPNotFound @@ -109,6 +109,25 @@ def get_err_response(code): return resp +def get_acl(account_name): + body = ('' + '' + '%s' + '' + '' + '' + '' + '%s' + '' + 'FULL_CONTROL' + '' + '' + '' % + (account_name, account_name)) + return Response(body=body, content_type="text/plain") + + class Controller(object): def __init__(self, app): self.app = app @@ -165,6 +184,7 @@ class BucketController(Controller): **kwargs): Controller.__init__(self, app) self.container_name = unquote(container_name) + self.account_name = unquote(account_name) env['HTTP_X_AUTH_TOKEN'] = token env['PATH_INFO'] = '/v1/%s/%s' % (account_name, container_name) @@ -173,7 +193,7 @@ class BucketController(Controller): Handle GET Bucket (List Objects) request """ if 'QUERY_STRING' in env: - args = dict(cgi.parse_qsl(env['QUERY_STRING'])) + args = dict(urlparse.parse_qsl(env['QUERY_STRING'], 1)) else: args = {} max_keys = min(int(args.get('max-keys', MAX_BUCKET_LISTING)), @@ -197,6 +217,9 @@ class BucketController(Controller): else: return get_err_response('InvalidURI') + if 'acl' in args: + return get_acl(self.account_name) + objects = loads(''.join(list(body_iter))) body = ('' '"%s"' \ + '' % headers['etag'] + return Response(status=200, body=body) + return Response(status=200, etag=headers['etag']) def DELETE(self, env, start_response): diff --git a/swift/container/server.py b/swift/container/server.py index 3601594afc..3ced13d61f 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -349,7 +349,7 @@ class ContainerController(object): if self.mount_check and not check_mount(self.root, drive): return Response(status='507 %s is not mounted' % drive) try: - args = simplejson.load(req.body_file) + args = simplejson.load(req.environ['wsgi.input']) except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain') ret = self.replicator_rpc.dispatch(post_args, args) diff --git a/swift/obj/server.py b/swift/obj/server.py index 6f8cdeaa01..131c11db69 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -503,8 +503,8 @@ class ObjectController(object): with file.mkstemp() as (fd, tmppath): if 'content-length' in request.headers: fallocate(fd, int(request.headers['content-length'])) - for chunk in iter(lambda: request.body_file.read( - self.network_chunk_size), ''): + reader = request.environ['wsgi.input'].read + for chunk in iter(lambda: reader(self.network_chunk_size), ''): upload_size += len(chunk) if time.time() > upload_expiration: return HTTPRequestTimeout(request=request) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 0a0991c517..cdb6b335db 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -929,8 +929,8 @@ class ObjectController(Controller): error_response = check_object_creation(req, self.object_name) if error_response: return error_response - data_source = \ - iter(lambda: req.body_file.read(self.app.client_chunk_size), '') + reader = req.environ['wsgi.input'].read + data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') if source_header: source_header = unquote(source_header) diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py index ce9694e4e6..f2d5bc124a 100644 --- a/swift/stats/log_processor.py +++ b/swift/stats/log_processor.py @@ -32,7 +32,8 @@ from swift.common.daemon import Daemon class BadFileDownload(Exception): - pass + def __init__(self, status_code=None): + self.status_code = status_code class LogProcessor(object): @@ -161,7 +162,7 @@ class LogProcessor(object): code, o = self.internal_proxy.get_object(swift_account, container_name, object_name) if code < 200 or code >= 300: - raise BadFileDownload() + raise BadFileDownload(code) last_part = '' last_compressed_part = '' # magic in the following zlib.decompressobj argument is courtesy of @@ -271,8 +272,13 @@ class LogProcessorDaemon(Daemon): already_processed_files = cPickle.loads(buf) else: already_processed_files = set() - except BadFileDownload: - already_processed_files = set() + except BadFileDownload, err: + if err.status_code == 404: + already_processed_files = set() + else: + self.logger.error(_('Log processing unable to load list of ' + 'already processed log files')) + return self.logger.debug(_('found %d processed files') % \ len(already_processed_files)) logs_to_process = self.log_processor.get_data_list(lookback_start, diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index 64a1801af2..a9248c7c17 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -18,7 +18,7 @@ import os import hashlib import time import gzip -import glob +import re from paste.deploy import appconfig from swift.common.internal_proxy import InternalProxy @@ -44,29 +44,30 @@ class LogUploader(Daemon): def __init__(self, uploader_conf, plugin_name): super(LogUploader, self).__init__(uploader_conf) - log_dir = uploader_conf.get('log_dir', '/var/log/swift/') - swift_account = uploader_conf['swift_account'] - container_name = uploader_conf['container_name'] - source_filename_format = uploader_conf['source_filename_format'] + log_name = '%s-log-uploader' % plugin_name + self.logger = utils.get_logger(uploader_conf, log_name, + log_route=plugin_name) + self.log_dir = uploader_conf.get('log_dir', '/var/log/swift/') + self.swift_account = uploader_conf['swift_account'] + self.container_name = uploader_conf['container_name'] proxy_server_conf_loc = uploader_conf.get('proxy_server_conf', '/etc/swift/proxy-server.conf') proxy_server_conf = appconfig('config:%s' % proxy_server_conf_loc, name='proxy-server') - new_log_cutoff = int(uploader_conf.get('new_log_cutoff', '7200')) - unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \ - ('true', 'on', '1', 'yes') - self.unlink_log = unlink_log - self.new_log_cutoff = new_log_cutoff - if not log_dir.endswith('/'): - log_dir = log_dir + '/' - self.log_dir = log_dir - self.swift_account = swift_account - self.container_name = container_name - self.filename_format = source_filename_format self.internal_proxy = InternalProxy(proxy_server_conf) - log_name = '%s-log-uploader' % plugin_name - self.logger = utils.get_logger(uploader_conf, log_name, - log_route=plugin_name) + self.new_log_cutoff = int(uploader_conf.get('new_log_cutoff', '7200')) + self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \ + utils.TRUE_VALUES + + # source_filename_format is deprecated + source_filename_format = uploader_conf.get('source_filename_format') + source_filename_pattern = uploader_conf.get('source_filename_pattern') + if source_filename_format and not source_filename_pattern: + self.logger.warning(_('source_filename_format is unreliable and ' + 'deprecated; use source_filename_pattern')) + self.pattern = self.convert_glob_to_regex(source_filename_format) + else: + self.pattern = source_filename_pattern or '%Y%m%d%H' def run_once(self, *args, **kwargs): self.logger.info(_("Uploading logs")) @@ -75,70 +76,114 @@ class LogUploader(Daemon): self.logger.info(_("Uploading logs complete (%0.2f minutes)") % ((time.time() - start) / 60)) + def convert_glob_to_regex(self, glob): + """ + Make a best effort to support old style config globs + + :param : old style config source_filename_format + + :returns : new style config source_filename_pattern + """ + pattern = glob + pattern = pattern.replace('.', r'\.') + pattern = pattern.replace('*', r'.*') + pattern = pattern.replace('?', r'.?') + return pattern + + def validate_filename_pattern(self): + """ + Validate source_filename_pattern + + :returns : valid regex pattern based on soruce_filename_pattern with + group matches substituded for date fmt markers + """ + pattern = self.pattern + markers = { + '%Y': ('year', '(?P[0-9]{4})'), + '%m': ('month', '(?P[0-1][0-9])'), + '%d': ('day', '(?P[0-3][0-9])'), + '%H': ('hour', '(?P[0-2][0-9])'), + } + for marker, (mtype, group) in markers.items(): + if marker not in self.pattern: + self.logger.error(_('source_filename_pattern much contain a ' + 'marker %(marker)s to match the ' + '%(mtype)s') % {'marker': marker, + 'mtype': mtype}) + return + pattern = pattern.replace(marker, group) + return pattern + + def get_relpath_to_files_under_log_dir(self): + """ + Look under log_dir recursively and return all filenames as relpaths + + :returns : list of strs, the relpath to all filenames under log_dir + """ + all_files = [] + for path, dirs, files in os.walk(self.log_dir): + all_files.extend(os.path.join(path, f) for f in files) + return [os.path.relpath(f, start=self.log_dir) for f in all_files] + + def filter_files(self, all_files, pattern): + """ + Filter files based on regex pattern + + :param all_files: list of strs, relpath of the filenames under log_dir + :param pattern: regex pattern to match against filenames + + :returns : dict mapping full path of file to match group dict + """ + filename2match = {} + found_match = False + for filename in all_files: + match = re.match(pattern, filename) + if match: + found_match = True + full_path = os.path.join(self.log_dir, filename) + filename2match[full_path] = match.groupdict() + else: + self.logger.debug(_('%(filename)s does not match ' + '%(pattern)s') % {'filename': filename, + 'pattern': pattern}) + return filename2match + def upload_all_logs(self): - i = [(self.filename_format.index(c), c) for c in '%Y %m %d %H'.split()] - i.sort() - year_offset = month_offset = day_offset = hour_offset = None - base_offset = len(self.log_dir.rstrip('/')) + 1 - for start, c in i: - offset = base_offset + start - if c == '%Y': - year_offset = offset, offset + 4 - # Add in the difference between len(%Y) and the expanded - # version of %Y (????). This makes sure the codes after this - # one will align properly in the final filename. - base_offset += 2 - elif c == '%m': - month_offset = offset, offset + 2 - elif c == '%d': - day_offset = offset, offset + 2 - elif c == '%H': - hour_offset = offset, offset + 2 - if not (year_offset and month_offset and day_offset and hour_offset): - # don't have all the parts, can't upload anything + """ + Match files under log_dir to source_filename_pattern and upload to swift + """ + pattern = self.validate_filename_pattern() + if not pattern: + self.logger.error(_('Invalid filename_format')) return - glob_pattern = self.filename_format - glob_pattern = glob_pattern.replace('%Y', '????', 1) - glob_pattern = glob_pattern.replace('%m', '??', 1) - glob_pattern = glob_pattern.replace('%d', '??', 1) - glob_pattern = glob_pattern.replace('%H', '??', 1) - filelist = glob.iglob(os.path.join(self.log_dir, glob_pattern)) - current_hour = int(time.strftime('%H')) - today = int(time.strftime('%Y%m%d')) - self.internal_proxy.create_container(self.swift_account, - self.container_name) - for filename in filelist: - try: - # From the filename, we need to derive the year, month, day, - # and hour for the file. These values are used in the uploaded - # object's name, so they should be a reasonably accurate - # representation of the time for which the data in the file was - # collected. The file's last modified time is not a reliable - # representation of the data in the file. For example, an old - # log file (from hour A) may be uploaded or moved into the - # log_dir in hour Z. The file's modified time will be for hour - # Z, and therefore the object's name in the system will not - # represent the data in it. - # If the filename doesn't match the format, it shouldn't be - # uploaded. - year = filename[slice(*year_offset)] - month = filename[slice(*month_offset)] - day = filename[slice(*day_offset)] - hour = filename[slice(*hour_offset)] - except IndexError: - # unexpected filename format, move on - self.logger.error(_("Unexpected log: %s") % filename) + all_files = self.get_relpath_to_files_under_log_dir() + filename2match = self.filter_files(all_files, pattern) + if not filename2match: + self.logger.info(_('No files in %(log_dir)s match %(pattern)s') % + {'log_dir': self.log_dir, 'pattern': pattern}) + return + if not self.internal_proxy.create_container(self.swift_account, + self.container_name): + self.logger.error(_('Unable to create container for ' + '%(account)s/%(container)s') % { + 'account': self.swift_account, + 'container': self.container_name}) + return + for filename, match in filename2match.items(): + # don't process very new logs + seconds_since_mtime = time.time() - os.stat(filename).st_mtime + if seconds_since_mtime < self.new_log_cutoff: + self.logger.debug(_("Skipping log: %(file)s " + "(< %(cutoff)d seconds old)") % { + 'file': filename, + 'cutoff': self.new_log_cutoff}) continue - if ((time.time() - os.stat(filename).st_mtime) < - self.new_log_cutoff): - # don't process very new logs - self.logger.debug( - _("Skipping log: %(file)s (< %(cutoff)d seconds old)") % - {'file': filename, 'cutoff': self.new_log_cutoff}) - continue - self.upload_one_log(filename, year, month, day, hour) + self.upload_one_log(filename, **match) def upload_one_log(self, filename, year, month, day, hour): + """ + Upload one file to swift + """ if os.path.getsize(filename) == 0: self.logger.debug(_("Log %s is 0 length, skipping") % filename) return diff --git a/test/functional/sample.conf b/test/functional/sample.conf index 602d7a4f29..00009c6db0 100644 --- a/test/functional/sample.conf +++ b/test/functional/sample.conf @@ -1,13 +1,9 @@ [func_test] # sample config auth_host = 127.0.0.1 -# For DevAuth: -auth_port = 11000 -# For Swauth: -# auth_port = 8080 +auth_port = 8080 auth_ssl = no -# For Swauth: -# auth_prefix = /auth/ +auth_prefix = /auth/ # Primary functional test account (needs admin access to the account) account = test diff --git a/test/probe/common.py b/test/probe/common.py index 08e8309a4b..b87699e8c8 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -25,24 +25,15 @@ from swift.common.ring import Ring SUPER_ADMIN_KEY = None -AUTH_TYPE = None c = ConfigParser() -AUTH_SERVER_CONF_FILE = environ.get('SWIFT_AUTH_SERVER_CONF_FILE', - '/etc/swift/auth-server.conf') -if c.read(AUTH_SERVER_CONF_FILE): - conf = dict(c.items('app:auth-server')) - SUPER_ADMIN_KEY = conf.get('super_admin_key', 'devauth') - AUTH_TYPE = 'devauth' +PROXY_SERVER_CONF_FILE = environ.get('SWIFT_PROXY_SERVER_CONF_FILE', + '/etc/swift/proxy-server.conf') +if c.read(PROXY_SERVER_CONF_FILE): + conf = dict(c.items('filter:swauth')) + SUPER_ADMIN_KEY = conf.get('super_admin_key', 'swauthkey') else: - PROXY_SERVER_CONF_FILE = environ.get('SWIFT_PROXY_SERVER_CONF_FILE', - '/etc/swift/proxy-server.conf') - if c.read(PROXY_SERVER_CONF_FILE): - conf = dict(c.items('filter:swauth')) - SUPER_ADMIN_KEY = conf.get('super_admin_key', 'swauthkey') - AUTH_TYPE = 'swauth' - else: - exit('Unable to read config file: %s' % AUTH_SERVER_CONF_FILE) + exit('Unable to read config file: %s' % PROXY_SERVER_CONF_FILE) def kill_pids(pids): @@ -57,9 +48,6 @@ def reset_environment(): call(['resetswift']) pids = {} try: - if AUTH_TYPE == 'devauth': - pids['auth'] = Popen(['swift-auth-server', - '/etc/swift/auth-server.conf']).pid pids['proxy'] = Popen(['swift-proxy-server', '/etc/swift/proxy-server.conf']).pid port2server = {} @@ -73,21 +61,9 @@ def reset_environment(): container_ring = Ring('/etc/swift/container.ring.gz') object_ring = Ring('/etc/swift/object.ring.gz') sleep(5) - if AUTH_TYPE == 'devauth': - conn = http_connect('127.0.0.1', '11000', 'POST', - '/recreate_accounts', - headers={'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': SUPER_ADMIN_KEY}) - resp = conn.getresponse() - if resp.status != 200: - raise Exception('Recreating accounts failed. (%d)' % - resp.status) - url, token = get_auth('http://127.0.0.1:11000/auth', 'test:tester', - 'testing') - elif AUTH_TYPE == 'swauth': - call(['recreateaccounts']) - url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', - 'test:tester', 'testing') + call(['recreateaccounts']) + url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', + 'test:tester', 'testing') account = url.split('/')[-1] except BaseException, err: kill_pids(pids) diff --git a/test/unit/auth/__init__.py b/test/unit/auth/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/test/unit/auth/test_server.py b/test/unit/auth/test_server.py deleted file mode 100644 index d58556ab22..0000000000 --- a/test/unit/auth/test_server.py +++ /dev/null @@ -1,977 +0,0 @@ -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import with_statement -import unittest -import os -from shutil import rmtree -from StringIO import StringIO -from uuid import uuid4 -from logging import StreamHandler - -import sqlite3 -from webob import Request - -from swift.auth import server as auth_server -from swift.common.db import DatabaseConnectionError, get_db_connection -from swift.common.utils import get_logger - - -class TestException(Exception): - pass - - -def fake_http_connect(*code_iter, **kwargs): - class FakeConn(object): - def __init__(self, status): - self.status = status - self.reason = 'Fake' - self.host = '1.2.3.4' - self.port = '1234' - def getresponse(self): - if 'slow' in kwargs: - sleep(0.2) - if 'raise_exc' in kwargs: - raise kwargs['raise_exc'] - return self - def getheaders(self): - return {'x-account-bytes-used': '20'} - def read(self, amt=None): - return '' - def getheader(self, name): - return self.getheaders().get(name.lower()) - code_iter = iter(code_iter) - def connect(*args, **kwargs): - connect.last_args = args - connect.last_kwargs = kwargs - return FakeConn(code_iter.next()) - return connect - - -class TestAuthServer(unittest.TestCase): - - def setUp(self): - self.ohttp_connect = auth_server.http_connect - self.testdir = os.path.join(os.path.dirname(__file__), - 'auth_server') - rmtree(self.testdir, ignore_errors=1) - os.mkdir(self.testdir) - self.conf = {'swift_dir': self.testdir, 'log_name': 'auth', - 'super_admin_key': 'testkey'} - self.controller = auth_server.AuthController(self.conf) - - def tearDown(self): - auth_server.http_connect = self.ohttp_connect - rmtree(self.testdir, ignore_errors=1) - - def test_get_conn(self): - with self.controller.get_conn() as conn: - pass - exc = False - try: - with self.controller.get_conn() as conn: - raise TestException('test') - except TestException: - exc = True - self.assert_(exc) - # We allow reentrant calls for the auth-server - with self.controller.get_conn() as conn1: - exc = False - try: - with self.controller.get_conn() as conn2: - self.assert_(conn1 is not conn2) - except DatabaseConnectionError: - exc = True - self.assert_(not exc) - self.controller.conn = None - with self.controller.get_conn() as conn: - self.assert_(conn is not None) - - def test_validate_token_non_existant_token(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing',).split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - self.assertEquals(self.controller.validate_token(token + 'bad'), False) - - def test_validate_token_good(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing',).split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - - def test_validate_token_expired(self): - orig_time = auth_server.time - try: - auth_server.time = lambda: 1 - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user('test', 'tester', - 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - auth_server.time = lambda: 1 + self.controller.token_life - self.assertEquals(self.controller.validate_token(token), False) - finally: - auth_server.time = orig_time - - def test_create_user_no_new_account(self): - auth_server.http_connect = fake_http_connect(201) - result = self.controller.create_user('', 'tester', 'testing') - self.assertFalse(result) - - def test_create_user_no_new_user(self): - auth_server.http_connect = fake_http_connect(201) - result = self.controller.create_user('test', '', 'testing') - self.assertFalse(result) - - def test_create_user_no_new_password(self): - auth_server.http_connect = fake_http_connect(201) - result = self.controller.create_user('test', 'tester', '') - self.assertFalse(result) - - def test_create_user_good(self): - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test', 'tester', 'testing') - self.assert_(url) - self.assertEquals('/'.join(url.split('/')[:-1]), - self.controller.default_cluster_url.rstrip('/'), repr(url)) - - def test_recreate_accounts_none(self): - auth_server.http_connect = fake_http_connect(201) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '0', repr(rv)) - self.assertEquals(rv.split()[-1], '[]', repr(rv)) - - def test_recreate_accounts_one(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '1', repr(rv)) - self.assertEquals(rv.split()[-1], '[]', repr(rv)) - - def test_recreate_accounts_several(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test1', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test2', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test3', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test4', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201, 201, 201, 201) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '4', repr(rv)) - self.assertEquals(rv.split()[-1], '[]', repr(rv)) - - def test_recreate_accounts_one_fail(self): - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test', 'tester', 'testing') - cfaccount = url.split('/')[-1] - auth_server.http_connect = fake_http_connect(500) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '1', repr(rv)) - self.assertEquals(rv.split()[-1], '[%s]' % repr(cfaccount), - repr(rv)) - - def test_recreate_accounts_several_fail(self): - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test1', 'tester', 'testing') - cfaccounts = [url.split('/')[-1]] - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test2', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test3', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test4', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(500, 500, 500, 500) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '4', repr(rv)) - failed = rv.split('[', 1)[-1][:-1].split(', ') - self.assertEquals(set(failed), set(repr(a) for a in cfaccounts)) - - def test_recreate_accounts_several_fail_some(self): - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test1', 'tester', 'testing') - cfaccounts = [url.split('/')[-1]] - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test2', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test3', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test4', 'tester', 'testing') - cfaccounts.append(url.split('/')[-1]) - auth_server.http_connect = fake_http_connect(500, 201, 500, 201) - rv = self.controller.recreate_accounts() - self.assertEquals(rv.split()[0], '4', repr(rv)) - failed = rv.split('[', 1)[-1][:-1].split(', ') - self.assertEquals( - len(set(repr(a) for a in cfaccounts) - set(failed)), 2) - - def test_auth_bad_path(self): - res = self.controller.handle_auth( - Request.blank('', environ={'REQUEST_METHOD': 'GET'})) - self.assertEquals(res.status_int, 400) - res = self.controller.handle_auth(Request.blank('/bad', - environ={'REQUEST_METHOD': 'GET'})) - self.assertEquals(res.status_int, 400) - - def test_auth_SOSO_missing_headers(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-Pass': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester'})) - self.assertEquals(res.status_int, 401) - - def test_auth_SOSO_bad_account(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/testbad/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1//auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_SOSO_bad_user(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'testerbad', - 'X-Storage-Pass': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': '', - 'X-Storage-Pass': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_SOSO_bad_password(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testingbad'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': ''})) - self.assertEquals(res.status_int, 401) - - def test_auth_SOSO_good(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - - def test_auth_SOSO_good_Mosso_headers(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:tester', - 'X-Auth-Key': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - - def test_auth_SOSO_bad_Mosso_headers(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing',).split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test2:tester', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': ':tester', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/v1/test/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_missing_headers(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:tester'})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_bad_header_format(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'badformat', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': '', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_bad_account(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'testbad:tester', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': ':tester', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_bad_user(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:testerbad', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:', - 'X-Auth-Key': 'testing'})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_bad_password(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:tester', - 'X-Auth-Key': 'testingbad'})) - self.assertEquals(res.status_int, 401) - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:tester', - 'X-Auth-Key': ''})) - self.assertEquals(res.status_int, 401) - - def test_auth_Mosso_good(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'test:tester', - 'X-Auth-Key': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - - def test_auth_Mosso_good_SOSO_header_names(self): - auth_server.http_connect = fake_http_connect(201) - cfaccount = self.controller.create_user( - 'test', 'tester', 'testing').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/auth', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Storage-User': 'test:tester', - 'X-Storage-Pass': 'testing'})) - token = res.headers['x-storage-token'] - ttl, _junk, _junk, _junk = self.controller.validate_token(token) - self.assert_(ttl > 0, repr(ttl)) - - def test_basic_logging(self): - log = StringIO() - log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth-server', log_route='auth-server') - logger.logger.addHandler(log_handler) - try: - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test', 'tester', 'testing') - self.assertEquals(log.getvalue().rsplit(' ', 1)[0], - "SUCCESS create_user('test', 'tester', _, False, False) " - "= %s" % repr(url)) - log.truncate(0) - def start_response(*args): - pass - self.controller.handleREST({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/v1/test/auth', - 'QUERY_STRING': 'test=True', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': StringIO(), - 'wsgi.errors': StringIO(), - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False, - 'HTTP_X_FORWARDED_FOR': 'testhost', - 'HTTP_X_STORAGE_USER': 'tester', - 'HTTP_X_STORAGE_PASS': 'testing'}, - start_response) - logsegs = log.getvalue().split(' [', 1) - logsegs[1:] = logsegs[1].split('] ', 1) - logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' - logsegs[2:] = logsegs[2].split(' ') - logsegs[-1] = '0.1234' - self.assertEquals(' '.join(logsegs), 'testhost - - ' - '[01/Jan/2001:01:02:03 +0000] "GET /v1/test/auth?test=True ' - 'HTTP/1.0" 204 - "-" "-" - - - - - - - - - "-" "None" "-" ' - '0.1234') - self.controller.log_headers = True - log.truncate(0) - self.controller.handleREST({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/v1/test/auth', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': StringIO(), - 'wsgi.errors': StringIO(), - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False, - 'HTTP_X_STORAGE_USER': 'tester', - 'HTTP_X_STORAGE_PASS': 'testing'}, - start_response) - logsegs = log.getvalue().split(' [', 1) - logsegs[1:] = logsegs[1].split('] ', 1) - logsegs[1] = '[01/Jan/2001:01:02:03 +0000]' - logsegs[2:] = logsegs[2].split(' ') - logsegs[-1] = '0.1234' - self.assertEquals(' '.join(logsegs), 'None - - [01/Jan/2001:' - '01:02:03 +0000] "GET /v1/test/auth HTTP/1.0" 204 - "-" "-" - ' - '- - - - - - - - "-" "None" "Content-Length: 0\n' - 'X-Storage-User: tester\nX-Storage-Pass: testing" 0.1234') - finally: - logger.logger.handlers.remove(log_handler) - - def test_unhandled_exceptions(self): - def request_causing_exception(*args, **kwargs): - pass - def start_response(*args): - pass - orig_Request = auth_server.Request - log = StringIO() - log_handler = StreamHandler(log) - logger = get_logger(self.conf, 'auth-server', log_route='auth-server') - logger.logger.addHandler(log_handler) - try: - auth_server.Request = request_causing_exception - self.controller.handleREST({'REQUEST_METHOD': 'GET', - 'SCRIPT_NAME': '', - 'PATH_INFO': '/v1/test/auth', - 'SERVER_NAME': '127.0.0.1', - 'SERVER_PORT': '8080', - 'SERVER_PROTOCOL': 'HTTP/1.0', - 'CONTENT_LENGTH': '0', - 'wsgi.version': (1, 0), - 'wsgi.url_scheme': 'http', - 'wsgi.input': StringIO(), - 'wsgi.errors': StringIO(), - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.run_once': False, - 'HTTP_X_STORAGE_USER': 'tester', - 'HTTP_X_STORAGE_PASS': 'testing'}, - start_response) - self.assert_(log.getvalue().startswith( - 'ERROR Unhandled exception in ReST request'), - log.getvalue()) - log.truncate(0) - finally: - auth_server.Request = orig_Request - logger.logger.handlers.remove(log_handler) - - def test_upgrading_from_db1(self): - swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex - os.mkdir(swift_dir) - try: - # Create db1 - db_file = os.path.join(swift_dir, 'auth.db') - conn = get_db_connection(db_file, okay_to_create=True) - conn.execute('''CREATE TABLE IF NOT EXISTS account ( - account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account - ON account (account)''') - conn.execute('''CREATE TABLE IF NOT EXISTS token ( - cfaccount TEXT, token TEXT, created FLOAT)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_cfaccount - ON token (cfaccount)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created - ON token (created)''') - conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password) - VALUES ('act', 'url', 'cfa', 'usr', 'pas')''') - conn.execute('''INSERT INTO token (cfaccount, token, created) - VALUES ('cfa', 'tok', '1')''') - conn.commit() - conn.close() - # Upgrade to current db - conf = {'swift_dir': swift_dir, 'super_admin_key': 'testkey'} - exc = None - try: - auth_server.AuthController(conf) - except Exception, err: - exc = err - self.assert_(str(err).strip().startswith('THERE ARE ACCOUNTS IN ' - 'YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW RESELLER'), err) - # Check new items exist and are correct - conn = get_db_connection(db_file) - row = conn.execute('SELECT admin FROM account').fetchone() - self.assertEquals(row[0], 't') - row = conn.execute('SELECT user FROM token').fetchone() - self.assert_(not row) - finally: - rmtree(swift_dir) - - def test_upgrading_from_db2(self): - swift_dir = '/tmp/swift_test_auth_%s' % uuid4().hex - os.mkdir(swift_dir) - try: - # Create db1 - db_file = os.path.join(swift_dir, 'auth.db') - conn = get_db_connection(db_file, okay_to_create=True) - conn.execute('''CREATE TABLE IF NOT EXISTS account ( - account TEXT, url TEXT, cfaccount TEXT, - user TEXT, password TEXT, admin TEXT)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_account_account - ON account (account)''') - conn.execute('''CREATE TABLE IF NOT EXISTS token ( - token TEXT, created FLOAT, - account TEXT, user TEXT, cfaccount TEXT)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_token - ON token (token)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_created - ON token (created)''') - conn.execute('''CREATE INDEX IF NOT EXISTS ix_token_account - ON token (account)''') - conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password, admin) - VALUES ('act', 'url', 'cfa', 'us1', 'pas', '')''') - conn.execute('''INSERT INTO account - (account, url, cfaccount, user, password, admin) - VALUES ('act', 'url', 'cfa', 'us2', 'pas', 't')''') - conn.execute('''INSERT INTO token - (token, created, account, user, cfaccount) - VALUES ('tok', '1', 'act', 'us1', 'cfa')''') - conn.commit() - conn.close() - # Upgrade to current db - conf = {'swift_dir': swift_dir, 'super_admin_key': 'testkey'} - exc = None - try: - auth_server.AuthController(conf) - except Exception, err: - exc = err - self.assert_(str(err).strip().startswith('THERE ARE ACCOUNTS IN ' - 'YOUR auth.db THAT DO NOT BEGIN WITH YOUR NEW RESELLER'), err) - # Check new items exist and are correct - conn = get_db_connection(db_file) - row = conn.execute('''SELECT admin, reseller_admin - FROM account WHERE user = 'us1' ''').fetchone() - self.assert_(not row[0], row[0]) - self.assert_(not row[1], row[1]) - row = conn.execute('''SELECT admin, reseller_admin - FROM account WHERE user = 'us2' ''').fetchone() - self.assertEquals(row[0], 't') - self.assert_(not row[1], row[1]) - row = conn.execute('SELECT user FROM token').fetchone() - self.assert_(row) - finally: - rmtree(swift_dir) - - def test_create_user_twice(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('test', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - self.assertEquals( - self.controller.create_user('test', 'tester', 'testing'), - 'already exists') - req = Request.blank('/account/test/tester', - headers={'X-Auth-User-Key': 'testing'}) - resp = self.controller.handle_add_user(req) - self.assertEquals(resp.status_int, 409) - - - def test_create_2users_1account(self): - auth_server.http_connect = fake_http_connect(201) - url = self.controller.create_user('test', 'tester', 'testing') - auth_server.http_connect = fake_http_connect(201) - url2 = self.controller.create_user('test', 'tester2', 'testing2') - self.assertEquals(url, url2) - - def test_no_super_admin_key(self): - conf = {'swift_dir': self.testdir, 'log_name': 'auth'} - self.assertRaises(ValueError, auth_server.AuthController, conf) - conf['super_admin_key'] = 'testkey' - controller = auth_server.AuthController(conf) - self.assertEquals(controller.super_admin_key, conf['super_admin_key']) - - def test_add_storage_account(self): - auth_server.http_connect = fake_http_connect(201) - stgact = self.controller.add_storage_account() - self.assert_(stgact.startswith(self.controller.reseller_prefix), - stgact) - # Make sure token given is the expected single use token - token = auth_server.http_connect.last_args[-1]['X-Auth-Token'] - self.assert_(self.controller.validate_token(token)) - self.assert_(not self.controller.validate_token(token)) - auth_server.http_connect = fake_http_connect(201) - stgact = self.controller.add_storage_account('bob') - self.assertEquals(stgact, 'bob') - # Make sure token given is the expected single use token - token = auth_server.http_connect.last_args[-1]['X-Auth-Token'] - self.assert_(self.controller.validate_token(token)) - self.assert_(not self.controller.validate_token(token)) - - def test_regular_user(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('act', 'usr', 'pas').split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1.0', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'act:usr', 'X-Auth-Key': 'pas'})) - _junk, _junk, _junk, stgact = \ - self.controller.validate_token(res.headers['x-auth-token']) - self.assertEquals(stgact, '') - - def test_account_admin(self): - auth_server.http_connect = fake_http_connect(201) - stgact = self.controller.create_user( - 'act', 'usr', 'pas', admin=True).split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1.0', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'act:usr', 'X-Auth-Key': 'pas'})) - _junk, _junk, _junk, vstgact = \ - self.controller.validate_token(res.headers['x-auth-token']) - self.assertEquals(stgact, vstgact) - - def test_reseller_admin(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user( - 'act', 'usr', 'pas', reseller_admin=True).split('/')[-1] - res = self.controller.handle_auth(Request.blank('/v1.0', - environ={'REQUEST_METHOD': 'GET'}, - headers={'X-Auth-User': 'act:usr', 'X-Auth-Key': 'pas'})) - _junk, _junk, _junk, stgact = \ - self.controller.validate_token(res.headers['x-auth-token']) - self.assertEquals(stgact, '.reseller_admin') - - def test_is_account_admin(self): - req = Request.blank('/', headers={'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey'}) - self.assert_(self.controller.is_account_admin(req, 'any')) - req = Request.blank('/', headers={'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey2'}) - self.assert_(not self.controller.is_account_admin(req, 'any')) - req = Request.blank('/', headers={'X-Auth-Admin-User': '.super_admi', - 'X-Auth-Admin-Key': 'testkey'}) - self.assert_(not self.controller.is_account_admin(req, 'any')) - - auth_server.http_connect = fake_http_connect(201, 201) - self.controller.create_user( - 'act1', 'resadmin', 'pas', reseller_admin=True).split('/')[-1] - self.controller.create_user('act1', 'usr', 'pas').split('/')[-1] - self.controller.create_user( - 'act2', 'actadmin', 'pas', admin=True).split('/')[-1] - - req = Request.blank('/', headers={'X-Auth-Admin-User': 'act1:resadmin', - 'X-Auth-Admin-Key': 'pas'}) - self.assert_(self.controller.is_account_admin(req, 'any')) - self.assert_(self.controller.is_account_admin(req, 'act1')) - self.assert_(self.controller.is_account_admin(req, 'act2')) - - req = Request.blank('/', headers={'X-Auth-Admin-User': 'act1:usr', - 'X-Auth-Admin-Key': 'pas'}) - self.assert_(not self.controller.is_account_admin(req, 'any')) - self.assert_(not self.controller.is_account_admin(req, 'act1')) - self.assert_(not self.controller.is_account_admin(req, 'act2')) - - req = Request.blank('/', headers={'X-Auth-Admin-User': 'act2:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - self.assert_(not self.controller.is_account_admin(req, 'any')) - self.assert_(not self.controller.is_account_admin(req, 'act1')) - self.assert_(self.controller.is_account_admin(req, 'act2')) - - def test_handle_add_user_create_reseller_admin(self): - auth_server.http_connect = fake_http_connect(201) - self.controller.create_user('act', 'usr', 'pas') - self.controller.create_user('act', 'actadmin', 'pas', admin=True) - self.controller.create_user('act', 'resadmin', 'pas', - reseller_admin=True) - - req = Request.blank('/account/act/resadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Reseller-Admin': 'true'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/resadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Reseller-Admin': 'true', - 'X-Auth-Admin-User': 'act:usr', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/resadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Reseller-Admin': 'true', - 'X-Auth-Admin-User': 'act:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/resadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Reseller-Admin': 'true', - 'X-Auth-Admin-User': 'act:resadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/resadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Reseller-Admin': 'true', - 'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - def test_handle_add_user_create_account_admin(self): - auth_server.http_connect = fake_http_connect(201, 201) - self.controller.create_user('act', 'usr', 'pas') - self.controller.create_user('act', 'actadmin', 'pas', admin=True) - self.controller.create_user('act2', 'actadmin', 'pas', admin=True) - self.controller.create_user('act2', 'resadmin', 'pas', - reseller_admin=True) - - req = Request.blank('/account/act/actadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/actadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:usr', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/actadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act2:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/actadmin2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - req = Request.blank('/account/act/actadmin3', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act2:resadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - req = Request.blank('/account/act/actadmin4', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - def test_handle_add_user_create_normal_user(self): - auth_server.http_connect = fake_http_connect(201, 201) - self.controller.create_user('act', 'usr', 'pas') - self.controller.create_user('act', 'actadmin', 'pas', admin=True) - self.controller.create_user('act2', 'actadmin', 'pas', admin=True) - self.controller.create_user('act2', 'resadmin', 'pas', - reseller_admin=True) - - req = Request.blank('/account/act/usr2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/usr2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:usr', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/usr2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act2:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/account/act/usr2', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - req = Request.blank('/account/act/usr3', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act2:resadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - req = Request.blank('/account/act/usr4', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey'}) - resp = self.controller.handle_add_user(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - def test_handle_account_recreate_permissions(self): - auth_server.http_connect = fake_http_connect(201, 201) - self.controller.create_user('act', 'usr', 'pas') - self.controller.create_user('act', 'actadmin', 'pas', admin=True) - self.controller.create_user('act', 'resadmin', 'pas', - reseller_admin=True) - - req = Request.blank('/recreate_accounts', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true'}) - resp = self.controller.handle_account_recreate(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/recreate_accounts', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:usr', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_account_recreate(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/recreate_accounts', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:actadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_account_recreate(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/recreate_accounts', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': 'act:resadmin', - 'X-Auth-Admin-Key': 'pas'}) - resp = self.controller.handle_account_recreate(req) - self.assert_(resp.status_int // 100 == 4, resp.status_int) - - req = Request.blank('/recreate_accounts', - headers={'X-Auth-User-Key': 'pas', - 'X-Auth-User-Admin': 'true', - 'X-Auth-Admin-User': '.super_admin', - 'X-Auth-Admin-Key': 'testkey'}) - resp = self.controller.handle_account_recreate(req) - self.assert_(resp.status_int // 100 == 2, resp.status_int) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py deleted file mode 100644 index f6718f68bc..0000000000 --- a/test/unit/common/middleware/test_auth.py +++ /dev/null @@ -1,471 +0,0 @@ -# Copyright (c) 2010-2011 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import with_statement -import logging -import os -import sys -import unittest -from contextlib import contextmanager - -import eventlet -from webob import Request - -from swift.common.middleware import auth - -# mocks -logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - - -class FakeMemcache(object): - def __init__(self): - self.store = {} - - def get(self, key): - return self.store.get(key) - - def set(self, key, value, timeout=0): - self.store[key] = value - return True - - def incr(self, key, timeout=0): - self.store[key] = self.store.setdefault(key, 0) + 1 - return self.store[key] - - @contextmanager - def soft_lock(self, key, timeout=0, retries=5): - yield True - - def delete(self, key): - try: - del self.store[key] - except Exception: - pass - return True - - -def mock_http_connect(response, headers=None, with_exc=False): - class FakeConn(object): - def __init__(self, status, headers, with_exc): - self.status = status - self.reason = 'Fake' - self.host = '1.2.3.4' - self.port = '1234' - self.with_exc = with_exc - self.headers = headers - if self.headers is None: - self.headers = {} - - def getresponse(self): - if self.with_exc: - raise Exception('test') - return self - - def getheader(self, header): - return self.headers[header] - - def read(self, amt=None): - return '' - - def close(self): - return - - return lambda *args, **kwargs: FakeConn(response, headers, with_exc) - - -class Logger(object): - - def __init__(self): - self.error_value = None - self.exception_value = None - - def error(self, msg, *args, **kwargs): - self.error_value = (msg, args, kwargs) - - def exception(self, msg, *args, **kwargs): - _junk, exc, _junk = sys.exc_info() - self.exception_value = (msg, - '%s %s' % (exc.__class__.__name__, str(exc)), args, kwargs) - - -class FakeApp(object): - - def __init__(self): - self.i_was_called = False - - def __call__(self, env, start_response): - self.i_was_called = True - req = Request.blank('', environ=env) - if 'swift.authorize' in env: - resp = env['swift.authorize'](req) - if resp: - return resp(env, start_response) - return ['204 No Content'] - - -def start_response(*args): - pass - - -class TestAuth(unittest.TestCase): - - def setUp(self): - self.test_auth = auth.filter_factory({})(FakeApp()) - - def test_auth_deny_non_reseller_prefix(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - reqenv = {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/BLAH_account', - 'HTTP_X_AUTH_TOKEN': 'BLAH_t', 'swift.cache': FakeMemcache()} - result = ''.join(self.test_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('401'), result) - self.assertEquals(reqenv['swift.authorize'], - self.test_auth.denied_response) - finally: - auth.http_connect = old_http_connect - - def test_auth_deny_non_reseller_prefix_no_override(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - fake_authorize = lambda x: lambda x, y: ['500 Fake'] - reqenv = {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/BLAH_account', - 'HTTP_X_AUTH_TOKEN': 'BLAH_t', 'swift.cache': FakeMemcache(), - 'swift.authorize': fake_authorize} - result = ''.join(self.test_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('500 Fake'), result) - self.assertEquals(reqenv['swift.authorize'], fake_authorize) - finally: - auth.http_connect = old_http_connect - - def test_auth_no_reseller_prefix_deny(self): - # Ensures that when we have no reseller prefix, we don't deny a request - # outright but set up a denial swift.authorize and pass the request on - # down the chain. - old_http_connect = auth.http_connect - try: - local_app = FakeApp() - local_auth = \ - auth.filter_factory({'reseller_prefix': ''})(local_app) - auth.http_connect = mock_http_connect(404) - reqenv = {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/account', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': FakeMemcache()} - result = ''.join(local_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('401'), result) - self.assert_(local_app.i_was_called) - self.assertEquals(reqenv['swift.authorize'], - local_auth.denied_response) - finally: - auth.http_connect = old_http_connect - - def test_auth_no_reseller_prefix_allow(self): - # Ensures that when we have no reseller prefix, we can still allow - # access if our auth server accepts requests - old_http_connect = auth.http_connect - try: - local_app = FakeApp() - local_auth = \ - auth.filter_factory({'reseller_prefix': ''})(local_app) - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - reqenv = {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/act', - 'HTTP_X_AUTH_TOKEN': 't', 'swift.cache': None} - result = ''.join(local_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - self.assert_(local_app.i_was_called) - self.assertEquals(reqenv['swift.authorize'], - local_auth.authorize) - finally: - auth.http_connect = old_http_connect - - def test_auth_no_reseller_prefix_no_token(self): - # Check that normally we set up a call back to our authorize. - local_auth = \ - auth.filter_factory({'reseller_prefix': ''})(FakeApp()) - reqenv = {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/v1/account', - 'swift.cache': FakeMemcache()} - result = ''.join(local_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('401'), result) - self.assertEquals(reqenv['swift.authorize'], local_auth.authorize) - # Now make sure we don't override an existing swift.authorize when we - # have no reseller prefix. - local_authorize = lambda req: None - reqenv['swift.authorize'] = local_authorize - result = ''.join(local_auth(reqenv, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - self.assertEquals(reqenv['swift.authorize'], local_authorize) - - def test_auth_fail(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(404) - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': FakeMemcache()}, - lambda x, y: None)) - self.assert_(result.startswith('401'), result) - finally: - auth.http_connect = old_http_connect - - def test_auth_success(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'PATH_INFO': '/v/AUTH_cfa', 'HTTP_X_AUTH_TOKEN': 'AUTH_t', - 'swift.cache': FakeMemcache()}, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - finally: - auth.http_connect = old_http_connect - - def test_auth_memcache(self): - old_http_connect = auth.http_connect - try: - fake_memcache = FakeMemcache() - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'PATH_INFO': '/v/AUTH_cfa', 'HTTP_X_AUTH_TOKEN': 'AUTH_t', - 'swift.cache': fake_memcache}, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - auth.http_connect = mock_http_connect(404) - # Should still be in memcache - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'PATH_INFO': '/v/AUTH_cfa', 'HTTP_X_AUTH_TOKEN': 'AUTH_t', - 'swift.cache': fake_memcache}, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - finally: - auth.http_connect = old_http_connect - - def test_auth_just_expired(self): - old_http_connect = auth.http_connect - try: - fake_memcache = FakeMemcache() - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '0', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'PATH_INFO': '/v/AUTH_cfa', 'HTTP_X_AUTH_TOKEN': 'AUTH_t', - 'swift.cache': fake_memcache}, lambda x, y: None)) - self.assert_(result.startswith('204'), result) - auth.http_connect = mock_http_connect(404) - # Should still be in memcache, but expired - result = ''.join(self.test_auth({'REQUEST_METHOD': 'GET', - 'HTTP_X_AUTH_TOKEN': 'AUTH_t', 'swift.cache': fake_memcache}, - lambda x, y: None)) - self.assert_(result.startswith('401'), result) - finally: - auth.http_connect = old_http_connect - - def test_middleware_success(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - req = Request.blank('/v/AUTH_cfa/c/o', - headers={'x-auth-token': 'AUTH_t'}) - req.environ['swift.cache'] = FakeMemcache() - result = ''.join(self.test_auth(req.environ, start_response)) - self.assert_(result.startswith('204'), result) - self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') - finally: - auth.http_connect = old_http_connect - - def test_middleware_no_header(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - req = Request.blank('/v/AUTH_cfa/c/o') - req.environ['swift.cache'] = FakeMemcache() - result = ''.join(self.test_auth(req.environ, start_response)) - self.assert_(result.startswith('401'), result) - self.assert_(not req.remote_user, req.remote_user) - finally: - auth.http_connect = old_http_connect - - def test_middleware_storage_token(self): - old_http_connect = auth.http_connect - try: - auth.http_connect = mock_http_connect(204, - {'x-auth-ttl': '1234', 'x-auth-groups': 'act:usr,act,AUTH_cfa'}) - req = Request.blank('/v/AUTH_cfa/c/o', - headers={'x-storage-token': 'AUTH_t'}) - req.environ['swift.cache'] = FakeMemcache() - result = ''.join(self.test_auth(req.environ, start_response)) - self.assert_(result.startswith('204'), result) - self.assertEquals(req.remote_user, 'act:usr,act,AUTH_cfa') - finally: - auth.http_connect = old_http_connect - - def test_authorize_bad_path(self): - req = Request.blank('/badpath') - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 401) - req = Request.blank('/badpath') - req.remote_user = 'act:usr,act,AUTH_cfa' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - req = Request.blank('') - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 404) - req = Request.blank('') - req.environ['swift.cache'] = FakeMemcache() - result = ''.join(self.test_auth(req.environ, lambda x, y: None)) - self.assert_(result.startswith('404'), result) - - def test_authorize_account_access(self): - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act,AUTH_cfa' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - def test_authorize_acl_group_access(self): - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act:usr' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act2' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = 'act:usr2' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - def test_deny_cross_reseller(self): - # Tests that cross-reseller is denied, even if ACLs/group names match - req = Request.blank('/v1/OTHER_cfa') - req.remote_user = 'act:usr,act,AUTH_cfa' - req.acl = 'act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - def test_authorize_acl_referrer_access(self): - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = '.r:*' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.acl = '.r:.example.com' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - req = Request.blank('/v1/AUTH_cfa') - req.remote_user = 'act:usr,act' - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 401) - req = Request.blank('/v1/AUTH_cfa') - req.acl = '.r:*' - self.assertEquals(self.test_auth.authorize(req), None) - req = Request.blank('/v1/AUTH_cfa') - req.acl = '.r:.example.com' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 401) - req = Request.blank('/v1/AUTH_cfa') - req.referer = 'http://www.example.com/index.html' - req.acl = '.r:.example.com' - self.assertEquals(self.test_auth.authorize(req), None) - - def test_account_put_permissions(self): - req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) - req.remote_user = 'act:usr,act,AUTH_other' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - # Even PUTs to your own account as account admin should fail - req = Request.blank('/v1/AUTH_old', environ={'REQUEST_METHOD': 'PUT'}) - req.remote_user = 'act:usr,act,AUTH_old' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) - req.remote_user = 'act:usr,act,.reseller_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(resp, None) - - # .super_admin is not something the middleware should ever see or care - # about - req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) - req.remote_user = 'act:usr,act,.super_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - def test_account_delete_permissions(self): - req = Request.blank('/v1/AUTH_new', - environ={'REQUEST_METHOD': 'DELETE'}) - req.remote_user = 'act:usr,act' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - req = Request.blank('/v1/AUTH_new', - environ={'REQUEST_METHOD': 'DELETE'}) - req.remote_user = 'act:usr,act,AUTH_other' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - # Even DELETEs to your own account as account admin should fail - req = Request.blank('/v1/AUTH_old', - environ={'REQUEST_METHOD': 'DELETE'}) - req.remote_user = 'act:usr,act,AUTH_old' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - req = Request.blank('/v1/AUTH_new', - environ={'REQUEST_METHOD': 'DELETE'}) - req.remote_user = 'act:usr,act,.reseller_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(resp, None) - - # .super_admin is not something the middleware should ever see or care - # about - req = Request.blank('/v1/AUTH_new', - environ={'REQUEST_METHOD': 'DELETE'}) - req.remote_user = 'act:usr,act,.super_admin' - resp = self.test_auth.authorize(req) - self.assertEquals(resp and resp.status_int, 403) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/middleware/test_swift3.py b/test/unit/common/middleware/test_swift3.py index d91cda379f..2324782da8 100644 --- a/test/unit/common/middleware/test_swift3.py +++ b/test/unit/common/middleware/test_swift3.py @@ -421,6 +421,23 @@ class TestSwift3(unittest.TestCase): resp = local_app(req.environ, local_app.app.do_start_response) self.assertEquals(local_app.app.response_args[0].split()[0], '204') + def _check_acl(self, owner, resp): + dom = xml.dom.minidom.parseString("".join(resp)) + self.assertEquals(dom.firstChild.nodeName, 'AccessControlPolicy') + name = dom.getElementsByTagName('Permission')[0].childNodes[0].nodeValue + self.assertEquals(name, 'FULL_CONTROL') + name = dom.getElementsByTagName('ID')[0].childNodes[0].nodeValue + self.assertEquals(name, owner) + + def test_bucket_acl_GET(self): + local_app = swift3.filter_factory({})(FakeAppBucket()) + bucket_name = 'junk' + req = Request.blank('/%s?acl' % bucket_name, + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + resp = local_app(req.environ, local_app.app.do_start_response) + self._check_acl('test:tester', resp) + def _test_object_GETorHEAD(self, method): local_app = swift3.filter_factory({})(FakeAppObject()) req = Request.blank('/bucket/object', @@ -508,7 +525,7 @@ class TestSwift3(unittest.TestCase): self.assertEquals(app.req.headers['ETag'], '7dfa07a8e59ddbcd1dc84d4c4f82aea1') self.assertEquals(app.req.headers['X-Object-Meta-Something'], 'oh hai') - self.assertEquals(app.req.headers['X-Object-Copy'], '/some/source') + self.assertEquals(app.req.headers['X-Copy-From'], '/some/source') def test_object_DELETE_error(self): code = self._test_method_error(FakeAppObject, 'DELETE', @@ -529,6 +546,13 @@ class TestSwift3(unittest.TestCase): resp = local_app(req.environ, local_app.app.do_start_response) self.assertEquals(local_app.app.response_args[0].split()[0], '204') + def test_object_acl_GET(self): + local_app = swift3.filter_factory({})(FakeAppObject()) + req = Request.blank('/bucket/object?acl', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + resp = local_app(req.environ, local_app.app.do_start_response) + self._check_acl('test:tester', resp) if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index babcf99eca..782a31d2c5 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -835,9 +835,9 @@ class TestObjectController(unittest.TestCase): def test_status_map(statuses, expected): self.app.memcache.store = {} proxy_server.http_connect = mock_http_connect(*statuses) - req = Request.blank('/a/c/o.jpg', {}) + req = Request.blank('/a/c/o.jpg', + environ={'REQUEST_METHOD': 'PUT'}, body='some data') self.app.update_request(req) - req.body_file = StringIO('some data') res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) diff --git a/test/unit/stats/test_log_uploader.py b/test/unit/stats/test_log_uploader.py index b82e0ce02c..9b733624c6 100644 --- a/test/unit/stats/test_log_uploader.py +++ b/test/unit/stats/test_log_uploader.py @@ -13,14 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: More tests - import unittest import os from datetime import datetime from tempfile import mkdtemp from shutil import rmtree +from functools import partial +from collections import defaultdict +import random +import string +from test.unit import temptree from swift.stats import log_uploader import logging @@ -29,34 +32,62 @@ LOGGER = logging.getLogger() DEFAULT_GLOB = '%Y%m%d%H' +COMPRESSED_DATA = '\x1f\x8b\x08\x08\x87\xa5zM\x02\xffdata\x00KI,I\x04\x00c' \ + '\xf3\xf3\xad\x04\x00\x00\x00' + + +def mock_appconfig(*args, **kwargs): + pass + + +class MockInternalProxy(): + + def __init__(self, *args, **kwargs): + pass + + def create_container(self, *args, **kwargs): + return True + + def upload_file(self, *args, **kwargs): + return True + + +_orig_LogUploader = log_uploader.LogUploader + + +class MockLogUploader(_orig_LogUploader): + + def __init__(self, conf, logger=LOGGER): + conf['swift_account'] = conf.get('swift_account', '') + conf['container_name'] = conf.get('container_name', '') + conf['new_log_cutoff'] = conf.get('new_log_cutoff', '0') + conf['source_filename_format'] = conf.get( + 'source_filename_format', conf.get('filename_format')) + log_uploader.LogUploader.__init__(self, conf, 'plugin') + self.logger = logger + self.uploaded_files = [] + + def upload_one_log(self, filename, year, month, day, hour): + d = {'year': year, 'month': month, 'day': day, 'hour': hour} + self.uploaded_files.append((filename, d)) + _orig_LogUploader.upload_one_log(self, filename, year, month, + day, hour) + class TestLogUploader(unittest.TestCase): - def test_upload_all_logs(self): + def setUp(self): + # mock internal proxy + self._orig_InternalProxy = log_uploader.InternalProxy + self._orig_appconfig = log_uploader.appconfig + log_uploader.InternalProxy = MockInternalProxy + log_uploader.appconfig = mock_appconfig - class MockInternalProxy(): - - def create_container(self, *args, **kwargs): - pass - - class MonkeyLogUploader(log_uploader.LogUploader): - - def __init__(self, conf, logger=LOGGER): - self.log_dir = conf['log_dir'] - self.filename_format = conf.get('filename_format', - DEFAULT_GLOB) - self.new_log_cutoff = 0 - self.logger = logger - self.internal_proxy = MockInternalProxy() - self.swift_account = '' - self.container_name = '' - - self.uploaded_files = [] - - def upload_one_log(self, filename, year, month, day, hour): - d = {'year': year, 'month': month, 'day': day, 'hour': hour} - self.uploaded_files.append((filename, d)) + def tearDown(self): + log_uploader.appconfig = self._orig_appconfig + log_uploader.InternalProxy = self._orig_InternalProxy + def test_deprecated_glob_style_upload_all_logs(self): tmpdir = mkdtemp() try: today = datetime.now() @@ -72,7 +103,7 @@ class TestLogUploader(unittest.TestCase): open(os.path.join(tmpdir, ts), 'w').close() conf = {'log_dir': tmpdir} - uploader = MonkeyLogUploader(conf) + uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 24) for i, file_date in enumerate(sorted(uploader.uploaded_files)): @@ -112,7 +143,7 @@ class TestLogUploader(unittest.TestCase): 'log_dir': '%s/' % tmpdir, 'filename_format': 'swift-blah_98764.%Y%m%d-%H*.tar.gz', } - uploader = MonkeyLogUploader(conf) + uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 24) for i, file_date in enumerate(sorted(uploader.uploaded_files)): @@ -146,22 +177,201 @@ class TestLogUploader(unittest.TestCase): 'log_dir': tmpdir, 'filename_format': '*.%Y%m%d%H.log', } - uploader = MonkeyLogUploader(conf) + uploader = MockLogUploader(conf) uploader.upload_all_logs() self.assertEquals(len(uploader.uploaded_files), 24) - for i, file_date in enumerate(sorted(uploader.uploaded_files)): + fname_to_int = lambda x: int(os.path.basename(x[0]).split('.')[0]) + numerically = lambda x, y: cmp(fname_to_int(x), + fname_to_int(y)) + for i, file_date in enumerate(sorted(uploader.uploaded_files, + cmp=numerically)): d = {'year': year, 'month': month, 'day': day, 'hour': i} for k, v in d.items(): d[k] = '%0.2d' % v expected = (os.path.join(tmpdir, '%s.%s%0.2d.log' % (i, today_str, i)), d) - # TODO: support wildcards before the date pattern - # (i.e. relative offsets) - #print file_date - #self.assertEquals(file_date, expected) + self.assertEquals(file_date, expected) finally: rmtree(tmpdir) + def test_bad_pattern_in_config(self): + files = [datetime.now().strftime('%Y%m%d%H')] + with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: + # invalid pattern + conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%h'} # should be %H + uploader = MockLogUploader(conf) + self.assertFalse(uploader.validate_filename_pattern()) + uploader.upload_all_logs() + self.assertEquals(uploader.uploaded_files, []) + + conf = {'log_dir': t, 'source_filename_pattern': '%Y%m%d%H'} + uploader = MockLogUploader(conf) + self.assert_(uploader.validate_filename_pattern()) + uploader.upload_all_logs() + self.assertEquals(len(uploader.uploaded_files), 1) + + + # deprecated warning on source_filename_format + class MockLogger(): + + def __init__(self): + self.msgs = defaultdict(list) + + def log(self, level, msg): + self.msgs[level].append(msg) + + def __getattr__(self, attr): + return partial(self.log, attr) + + logger = MockLogger.logger = MockLogger() + + def mock_get_logger(*args, **kwargs): + return MockLogger.logger + + _orig_get_logger = log_uploader.utils.get_logger + try: + log_uploader.utils.get_logger = mock_get_logger + conf = {'source_filename_format': '%Y%m%d%H'} + uploader = MockLogUploader(conf, logger=logger) + self.assert_([m for m in logger.msgs['warning'] + if 'deprecated' in m]) + finally: + log_uploader.utils.get_logger = _orig_get_logger + + # convert source_filename_format to regex + conf = {'source_filename_format': 'pattern-*.%Y%m%d%H.*.gz'} + uploader = MockLogUploader(conf) + expected = r'pattern-.*\.%Y%m%d%H\..*\.gz' + self.assertEquals(uploader.pattern, expected) + + # use source_filename_pattern if we have the choice! + conf = { + 'source_filename_format': 'bad', + 'source_filename_pattern': 'good', + } + uploader = MockLogUploader(conf) + self.assertEquals(uploader.pattern, 'good') + + def test_pattern_upload_all_logs(self): + + # test empty dir + with temptree([]) as t: + conf = {'log_dir': t} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 0) + + def get_random_length_str(max_len=10, chars=string.ascii_letters): + return ''.join(random.choice(chars) for x in + range(random.randint(1, max_len))) + + template = 'prefix_%(random)s_%(digits)s.blah.' \ + '%(datestr)s%(hour)0.2d00-%(next_hour)0.2d00-%(number)s.gz' + pattern = r'prefix_.*_[0-9]+\.blah\.%Y%m%d%H00-[0-9]{2}00' \ + '-[0-9]?[0-9]\.gz' + files_that_should_match = [] + # add some files that match + for i in range(24): + fname = template % { + 'random': get_random_length_str(), + 'digits': get_random_length_str(16, string.digits), + 'datestr': datetime.now().strftime('%Y%m%d'), + 'hour': i, + 'next_hour': i + 1, + 'number': random.randint(0, 20), + } + files_that_should_match.append(fname) + + # add some files that don't match + files = list(files_that_should_match) + for i in range(24): + fname = template % { + 'random': get_random_length_str(), + 'digits': get_random_length_str(16, string.digits), + 'datestr': datetime.now().strftime('%Y%m'), + 'hour': i, + 'next_hour': i + 1, + 'number': random.randint(0, 20), + } + files.append(fname) + + for fname in files: + print fname + + with temptree(files, contents=[COMPRESSED_DATA] * len(files)) as t: + self.assertEquals(len(os.listdir(t)), 48) + conf = {'source_filename_pattern': pattern, 'log_dir': t} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(os.listdir(t)), 24) + self.assertEquals(len(uploader.uploaded_files), 24) + files_that_were_uploaded = set(x[0] for x in + uploader.uploaded_files) + for f in files_that_should_match: + self.assert_(os.path.join(t, f) in files_that_were_uploaded) + + def test_log_cutoff(self): + files = [datetime.now().strftime('%Y%m%d%H')] + with temptree(files) as t: + conf = {'log_dir': t, 'new_log_cutoff': '7200'} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 0) + conf = {'log_dir': t, 'new_log_cutoff': '0'} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 1) + + def test_create_container_fail(self): + files = [datetime.now().strftime('%Y%m%d%H')] + with temptree(files) as t: + conf = {'log_dir': t} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 1) + + with temptree(files) as t: + conf = {'log_dir': t} + uploader = MockLogUploader(conf) + # mock create_container to fail + uploader.internal_proxy.create_container = lambda *args: False + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 0) + + def test_unlink_log(self): + files = [datetime.now().strftime('%Y%m%d%H')] + with temptree(files, contents=[COMPRESSED_DATA]) as t: + conf = {'log_dir': t, 'unlink_log': 'false'} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 1) + # file still there + self.assertEquals(len(os.listdir(t)), 1) + + conf = {'log_dir': t, 'unlink_log': 'true'} + uploader = MockLogUploader(conf) + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 1) + # file gone + self.assertEquals(len(os.listdir(t)), 0) + + def test_upload_file_failed(self): + files = [datetime.now().strftime('%Y%m%d%H')] + with temptree(files, contents=[COMPRESSED_DATA]) as t: + conf = {'log_dir': t, 'unlink_log': 'true'} + uploader = MockLogUploader(conf) + + # mock upload_file to fail, and clean up mock + def mock_upload_file(self, *args, **kwargs): + uploader.uploaded_files.pop() + return False + uploader.internal_proxy.upload_file = mock_upload_file + uploader.run_once() + self.assertEquals(len(uploader.uploaded_files), 0) + # file still there + self.assertEquals(len(os.listdir(t)), 1) + + if __name__ == '__main__': unittest.main()