openstack/swift
Code Issues Proposed changes

merged with trunk

Browse Source
This commit is contained in:
John Dickinson 2010-11-02 14:43:37 -05:00
commit d155b75112
19 changed files with 231 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
doc/source/_ga/layout.html Normal file
View File

@ -0,0 +1,17 @@
{% extends "!layout.html" %}
{% block footer %}
{{ super() }}
<script type="text/javascript">
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type="text/javascript">
try {
var pageTracker = _gat._getTracker("UA-17511903-1");
pageTracker._setDomainName("none");
pageTracker._setAllowLinker(true);
pageTracker._trackPageview();
} catch(err) {}</script>
{% endblock %}

8
doc/source/conf.py
View File

@ -41,7 +41,13 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
todo_include_todos = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# Changing the path so that the Hudson build output contains GA code and the source
# docs do not contain the code so local, offline sphinx builds are "clean."
templates_path = []
if os.getenv('HUDSON_PUBLISH_DOCS'):
templates_path = ['_ga', '_templates']
else:
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'

3
doc/source/development_saio.rst
View File

@ -17,6 +17,9 @@ virtual machine will emulate running a four node Swift cluster.
* Create guest virtual machine from the Ubuntu image.
Additional information about setting up a Swift development snapshot on other distributions is
available on the wiki at http://wiki.openstack.org/SAIOInstructions.
-----------------------------------------
Installing dependencies and the core code
-----------------------------------------

3
doc/source/getting_started.rst
View File

@ -37,5 +37,4 @@ Production
----------
We do not have documentation yet on how to set up and configure Swift for a
production cluster, but hope to begin work on those soon.
production cluster, but hope to begin work on those soon.

2
doc/source/index.rst
View File

@ -40,7 +40,7 @@ Deployment:
.. toctree::
:maxdepth: 1
deployment_guide
admin_guide
debian_package_guide

2
doc/source/ratelimit.rst
View File

@ -4,7 +4,7 @@ Rate Limiting
Rate limiting in swift is implemented as a pluggable middleware. Rate
limiting is performed on requests that result in database writes to the
account and container sqlite dbs. It uses memcached and is dependant on
account and container sqlite dbs. It uses memcached and is dependent on
the proxy servers having highly synchronized time. The rate limits are
limited by the accuracy of the proxy server clocks.

26
swift/account/server.py
View File

@ -77,7 +77,12 @@ class AccountController(object):
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_path(unquote(req.path), 3, 4)
try:
drive, part, account, container = split_path(unquote(req.path),
3, 4)
except ValueError, err:
return HTTPBadRequest(body=str(err), content_type='text/plain',
request=req)
if self.mount_check and not check_mount(self.root, drive):
return Response(status='507 %s is not mounted' % drive)
broker = self._get_account_broker(drive, part, account)
@ -199,16 +204,15 @@ class AccountController(object):
except UnicodeDecodeError, err:
return HTTPBadRequest(body='parameters not utf8',
content_type='text/plain', request=req)
header_format = req.accept.first_match(['text/plain',
'application/json',
'application/xml'])
format = query_format if query_format else header_format
if format.startswith('application/'):
format = format[12:]
if query_format:
req.accept = 'application/%s' % query_format.lower()
out_content_type = req.accept.best_match(
['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
account_list = broker.list_containers_iter(limit, marker, prefix,
delimiter)
if format == 'json':
out_content_type = 'application/json'
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
@ -220,8 +224,7 @@ class AccountController(object):
json_out.append(json_pattern %
(name, object_count, bytes_used))
account_list = '[' + ','.join(json_out) + ']'
elif format == 'xml':
out_content_type = 'application/xml'
elif out_content_type.endswith('/xml'):
output_list = ['<?xml version="1.0" encoding="UTF-8"?>',
'<account name="%s">' % account]
for (name, object_count, bytes_used, is_subdir) in account_list:
@ -238,7 +241,6 @@ class AccountController(object):
else:
if not account_list:
return HTTPNoContent(request=req, headers=resp_headers)
out_content_type = 'text/plain'
account_list = '\n'.join(r[0] for r in account_list) + '\n'
ret = Response(body=account_list, request=req, headers=resp_headers)
ret.content_type = out_content_type

7
swift/auth/server.py
View File

@ -516,8 +516,11 @@ YOU HAVE A FEW OPTIONS:
:param request: A webob.Request instance.
"""
pathsegs = \
split_path(request.path, minsegs=1, maxsegs=3, rest_with_last=True)
try:
pathsegs = split_path(request.path, minsegs=1, maxsegs=3,
rest_with_last=True)
except ValueError:
return HTTPBadRequest()
if pathsegs[0] == 'v1' and pathsegs[2] == 'auth':
account = pathsegs[1]
user = request.headers.get('x-storage-user')

18
swift/common/middleware/auth.py
View File

@ -16,7 +16,7 @@
from time import time
from eventlet.timeout import Timeout
from webob.exc import HTTPForbidden, HTTPUnauthorized
from webob.exc import HTTPForbidden, HTTPUnauthorized, HTTPNotFound
from swift.common.bufferedhttp import http_connect_raw as http_connect
from swift.common.middleware.acl import clean_acl, parse_acl, referrer_allowed
@ -82,8 +82,11 @@ class DevAuth(object):
# With a non-empty reseller_prefix, I would like to be called
# back for anonymous access to accounts I know I'm the
# definitive auth for.
version, rest = split_path(env.get('PATH_INFO', ''),
1, 2, True)
try:
version, rest = split_path(env.get('PATH_INFO', ''),
1, 2, True)
except ValueError:
return HTTPNotFound()(env, start_response)
if rest and rest.startswith(self.reseller_prefix):
# Handle anonymous access to accounts I'm the definitive
# auth for.
@ -103,10 +106,10 @@ class DevAuth(object):
def get_groups(self, token, memcache_client=None):
"""
Get groups for the given token.
If memcache_client is set, token credentials will be cached
appropriately.
With a cache miss, or no memcache_client, the configurated external
authentication server will be queried for the group information.
@ -146,7 +149,10 @@ class DevAuth(object):
Returns None if the request is authorized to continue or a standard
WSGI response callable if not.
"""
version, account, container, obj = split_path(req.path, 1, 4, True)
try:
version, account, container, obj = split_path(req.path, 1, 4, True)
except ValueError:
return HTTPNotFound(request=req)
if not account or not account.startswith(self.reseller_prefix):
return self.denied_response(req)
user_groups = (req.remote_user or '').split(',')

6
swift/common/middleware/ratelimit.py
View File

@ -14,6 +14,7 @@
import time
import eventlet
from webob import Request, Response
from webob.exc import HTTPNotFound
from swift.common.utils import split_path, cache_from_env, get_logger
from swift.proxy.server import get_container_memcache_key
@ -204,7 +205,10 @@ class RateLimitMiddleware(object):
req = Request(env)
if self.memcache_client is None:
self.memcache_client = cache_from_env(env)
version, account, container, obj = split_path(req.path, 1, 4, True)
try:
version, account, container, obj = split_path(req.path, 1, 4, True)
except ValueError:
return HTTPNotFound()(env, start_response)
ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
if ratelimit_resp is None:
return self.app(env, start_response)

2
swift/common/utils.py
View File

@ -208,6 +208,7 @@ def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False):
trailing data, raises ValueError.
:returns: list of segments with a length of maxsegs (non-existant
segments will return as None)
:raises: ValueError if given an invalid path
"""
if not maxsegs:
maxsegs = minsegs
@ -622,6 +623,7 @@ def write_pickle(obj, dest, tmp):
os.fsync(fd)
renamer(tmppath, dest)
def audit_location_generator(devices, datadir, mount_check=True, logger=None):
'''
Given a devices path and a data directory, yield (path, device,

19
swift/container/server.py
View File

@ -278,16 +278,15 @@ class ContainerController(object):
except UnicodeDecodeError, err:
return HTTPBadRequest(body='parameters not utf8',
content_type='text/plain', request=req)
header_format = req.accept.first_match(['text/plain',
'application/json',
'application/xml'])
format = query_format if query_format else header_format
if format.startswith('application/'):
format = format[12:]
if query_format:
req.accept = 'application/%s' % query_format.lower()
out_content_type = req.accept.best_match(
['text/plain', 'application/json',
'application/xml', 'text/xml'],
default_match='text/plain')
container_list = broker.list_objects_iter(limit, marker, prefix,
delimiter, path)
if format == 'json':
out_content_type = 'application/json'
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
'"content_type":%s, "last_modified":"%s"']
json_pattern = '{' + ','.join(json_pattern) + '}'
@ -307,8 +306,7 @@ class ContainerController(object):
content_type,
created_at))
container_list = '[' + ','.join(json_out) + ']'
elif format == 'xml':
out_content_type = 'application/xml'
elif out_content_type.endswith('/xml'):
xml_output = []
for (name, created_at, size, content_type, etag) in container_list:
# escape name and format date here
@ -330,7 +328,6 @@ class ContainerController(object):
else:
if not container_list:
return HTTPNoContent(request=req, headers=resp_headers)
out_content_type = 'text/plain'
container_list = '\n'.join(r[0] for r in container_list) + '\n'
ret = Response(body=container_list, request=req, headers=resp_headers)
ret.content_type = out_content_type

8
swift/obj/server.py
View File

@ -534,8 +534,12 @@ class ObjectController(object):
Handle REPLICATE requests for the Swift Object Server. This is used
by the object replicator to get hashes for directories.
"""
device, partition, suffix = split_path(
unquote(request.path), 2, 3, True)
try:
device, partition, suffix = split_path(
unquote(request.path), 2, 3, True)
except ValueError, e:
return HTTPBadRequest(body=str(e), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return Response(status='507 %s is not mounted' % device)
path = os.path.join(self.devices, device, DATADIR, partition)

2
swift/proxy/server.py
View File

@ -1261,6 +1261,8 @@ class BaseApplication(object):
:param path: path from request
:returns: tuple of (controller class, path dictionary)
:raises: ValueError (thrown by split_path) id given invalid path
"""
version, account, container, obj = split_path(path, 1, 4, True)
d = dict(version=version,

11
swift/stats/access_processor.py
View File

@ -66,10 +66,13 @@ class AccessLogProcessor(object):
self.logger.debug('Bad server name: found "%s" expected "%s"' \
% (server, self.server_name))
return {}
(version,
account,
container_name,
object_name) = split_path(request, 2, 4, True)
try:
(version, account, container_name, object_name) = \
split_path(request, 2, 4, True)
except ValueError, e:
self.logger.debug(
'Invalid path: %s from data: %s' % (e, repr(raw_log)))
return {}
if container_name is not None:
container_name = container_name.split('?', 1)[0]
if object_name is not None:

9
test/functional/tests.py
View File

@ -170,6 +170,15 @@ class TestAccount(Base):
self.assert_status(412)
self.assert_body('Bad URL')
def testInvalidPath(self):
was_url = self.env.account.conn.storage_url
self.env.account.conn.storage_url = "/%s" % was_url
self.env.account.conn.make_request('GET')
try:
self.assert_status(404)
finally:
self.env.account.conn.storage_url = was_url
def testPUT(self):
self.env.account.conn.make_request('PUT')
self.assert_status([403, 405])

3
test/unit/auth/test_server.py
View File

@ -241,8 +241,9 @@ class TestAuthServer(unittest.TestCase):
len(set(repr(a) for a in cfaccounts) - set(failed)), 2)
def test_auth_bad_path(self):
self.assertRaises(ValueError, self.controller.handle_auth,
res = self.controller.handle_auth(
Request.blank('', environ={'REQUEST_METHOD': 'GET'}))
self.assertEquals(res.status_int, 400)
res = self.controller.handle_auth(Request.blank('/bad',
environ={'REQUEST_METHOD': 'GET'}))
self.assertEquals(res.status_int, 400)

20
test/unit/common/middleware/test_ratelimit.py
View File

@ -366,6 +366,26 @@ class TestRateLimit(unittest.TestCase):
time_took = time.time() - begin
self.assert_(round(time_took, 1) == .4)
def test_call_invalid_path(self):
env = {'REQUEST_METHOD': 'GET',
'SCRIPT_NAME': '',
'PATH_INFO': '//v1/AUTH_1234567890',
'SERVER_NAME': '127.0.0.1',
'SERVER_PORT': '80',
'swift.cache': FakeMemcache(),
'SERVER_PROTOCOL': 'HTTP/1.0'}
app = lambda *args, **kwargs: None
rate_mid = ratelimit.RateLimitMiddleware(app, {},
logger=FakeLogger())
class a_callable(object):
def __call__(self, *args, **kwargs):
pass
resp = rate_mid.__call__(env, a_callable())
self.assert_('404 Not Found' in resp[0])
if __name__ == '__main__':
unittest.main()

152
test/unit/container/test_server.py
View File

@ -479,18 +479,20 @@ class TestContainerController(unittest.TestCase):
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 412)
def test_GET_format(self):
def test_GET_json(self):
# make a container
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT',
req = Request.blank('/sda1/p/a/jsonc', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
# test an empty container
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req = Request.blank('/sda1/p/a/jsonc?format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
self.assertEquals(resp.status_int, 200)
self.assertEquals(eval(resp.body), [])
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/c/%s'%i, environ=
req = Request.blank('/sda1/p/a/jsonc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
@ -514,8 +516,88 @@ class TestContainerController(unittest.TestCase):
"bytes":0,
"content_type":"text/plain",
"last_modified":"1970-01-01T00:00:01"}]
req = Request.blank('/sda1/p/a/jsonc?format=json',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/json')
self.assertEquals(eval(resp.body), json_body)
for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/json;q=1.0', 'application/*'):
req = Request.blank('/sda1/p/a/jsonc',
environ={'REQUEST_METHOD': 'GET'})
req.accept = accept
resp = self.controller.GET(req)
self.assertEquals(eval(resp.body), json_body,
'Invalid body for Accept: %s' % accept)
self.assertEquals(resp.content_type, 'application/json',
'Invalid content_type for Accept: %s' % accept)
def test_GET_plain(self):
# make a container
req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
# test an empty container
req = Request.blank('/sda1/p/a/plainc', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.status_int, 204)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/plainc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
plain_body = '0\n1\n2\n'
req = Request.blank('/sda1/p/a/plainc',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'text/plain')
self.assertEquals(resp.body, plain_body)
for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=0.8', '*/*',
'text/plain,application/xml'):
req = Request.blank('/sda1/p/a/plainc',
environ={'REQUEST_METHOD': 'GET'})
req.accept = accept
resp = self.controller.GET(req)
self.assertEquals(resp.body, plain_body,
'Invalid body for Accept: %s' % accept)
self.assertEquals(resp.content_type, 'text/plain',
'Invalid content_type for Accept: %s' % accept)
# test conflicting formats
req = Request.blank('/sda1/p/a/plainc?format=plain',
environ={'REQUEST_METHOD': 'GET'})
req.accept = 'application/json'
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'text/plain')
self.assertEquals(resp.body, plain_body)
def test_GET_xml(self):
# make a container
req = Request.blank('/sda1/p/a/xmlc', environ={'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '0'})
resp = self.controller.PUT(req)
# fill the container
for i in range(3):
req = Request.blank('/sda1/p/a/xmlc/%s'%i, environ=
{'REQUEST_METHOD': 'PUT',
'HTTP_X_TIMESTAMP': '1',
'HTTP_X_CONTENT_TYPE': 'text/plain',
'HTTP_X_ETAG': 'x',
'HTTP_X_SIZE': 0})
resp = self.controller.PUT(req)
self.assertEquals(resp.status_int, 201)
xml_body = '<?xml version="1.0" encoding="UTF-8"?>\n' \
'<container name="c">' \
'<container name="xmlc">' \
'<object><name>0</name><hash>x</hash><bytes>0</bytes>' \
'<content_type>text/plain</content_type>' \
'<last_modified>1970-01-01T00:00:01' \
@ -529,46 +611,30 @@ class TestContainerController(unittest.TestCase):
'<last_modified>1970-01-01T00:00:01' \
'</last_modified></object>' \
'</container>'
plain_body = '0\n1\n2\n'
req = Request.blank('/sda1/p/a/c?format=json', environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/json')
result = eval(resp.body)
self.assertEquals(result, json_body)
req = Request.blank('/sda1/p/a/c?format=xml', environ={'REQUEST_METHOD': 'GET'})
# tests
req = Request.blank('/sda1/p/a/xmlc?format=xml',
environ={'REQUEST_METHOD': 'GET'})
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/xml')
result = resp.body
self.assertEquals(result, xml_body)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.accept = 'application/json'
self.assertEquals(resp.body, xml_body)
for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9',
'*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'):
req = Request.blank('/sda1/p/a/xmlc',
environ={'REQUEST_METHOD': 'GET'})
req.accept = xml_accept
resp = self.controller.GET(req)
self.assertEquals(resp.body, xml_body,
'Invalid body for Accept: %s' % xml_accept)
self.assertEquals(resp.content_type, 'application/xml',
'Invalid content_type for Accept: %s' % xml_accept)
req = Request.blank('/sda1/p/a/xmlc',
environ={'REQUEST_METHOD': 'GET'})
req.accept = 'text/xml'
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'application/json')
result = eval(resp.body)
self.assertEquals(result, json_body)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.accept = '*/*'
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'text/plain')
result = resp.body
self.assertEquals(result, plain_body)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.accept = 'application/*'
resp = self.controller.GET(req)
result = eval(resp.body)
self.assertEquals(result, json_body)
req = Request.blank('/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'})
req.accept = 'application/xml'
resp = self.controller.GET(req)
result = resp.body
self.assertEquals(result, xml_body)
# test conflicting formats
req = Request.blank('/sda1/p/a/c?format=plain', environ={'REQUEST_METHOD': 'GET'})
req.accept = 'application/json'
resp = self.controller.GET(req)
self.assertEquals(resp.content_type, 'text/plain')
result = resp.body
self.assertEquals(result, plain_body)
self.assertEquals(resp.content_type, 'text/xml')
self.assertEquals(resp.body, xml_body)
def test_GET_marker(self):
# make a container