openstack/swift
Code Issues Proposed changes

use simplejson to serialize acct/cont results

Browse Source 
I did some benchmarking, and it's just a lot faster to use simplejson
to serialize account and container listings.

Change-Id: Ib0f751d9a5b6f4020031e952986811f0be090cce
This commit is contained in:
Michael Barton 2012-09-02 23:55:53 -07:00
parent 54a290718c
commit 0bb5d6da13
2 changed files with 20 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
swift/account/server.py
View File

@ -27,13 +27,12 @@ from webob.exc import HTTPAccepted, HTTPBadRequest, \
HTTPCreated, HTTPForbidden, HTTPInternalServerError, \
HTTPMethodNotAllowed, HTTPNoContent, HTTPNotFound, \
HTTPPreconditionFailed, HTTPConflict
import simplejson
import swift.common.db
from swift.common.db import AccountBroker
from swift.common.utils import get_logger, get_param, hash_path, public, \
normalize_timestamp, split_path, storage_directory, TRUE_VALUES, \
validate_device_partition
validate_device_partition, json
from swift.common.constraints import ACCOUNT_LISTING_LIMIT, \
check_mount, check_float, check_utf8, FORMAT2CONTENT_TYPE
from swift.common.db_replicator import ReplicatorRpc
@ -266,17 +265,14 @@ class AccountController(object):
account_list = broker.list_containers_iter(limit, marker, end_marker,
prefix, delimiter)
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
data = []
for (name, object_count, bytes_used, is_subdir) in account_list:
name = simplejson.dumps(name)
if is_subdir:
json_out.append('{"subdir":%s}' % name)
data.append({'subdir': name})
else:
json_out.append(json_pattern %
(name, object_count, bytes_used))
account_list = '[' + ','.join(json_out) + ']'
data.append({'name': name, 'count': object_count,
'bytes': bytes_used})
account_list = json.dumps(data)
elif out_content_type.endswith('/xml'):
output_list = ['<?xml version="1.0" encoding="UTF-8"?>',
'<account name="%s">' % account]
@ -321,7 +317,7 @@ class AccountController(object):
self.logger.increment('REPLICATE.errors')
return HTTPInsufficientStorage(drive=drive, request=req)
try:
args = simplejson.load(req.environ['wsgi.input'])
args = json.load(req.environ['wsgi.input'])
except ValueError, err:
self.logger.increment('REPLICATE.errors')
return HTTPBadRequest(body=str(err), content_type='text/plain')

35
swift/container/server.py
View File

@ -22,7 +22,6 @@ from urllib import unquote
from xml.sax import saxutils
from datetime import datetime
import simplejson
from eventlet import Timeout
from webob import Request, Response
from webob.exc import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
@ -33,7 +32,7 @@ import swift.common.db
from swift.common.db import ContainerBroker
from swift.common.utils import get_logger, get_param, hash_path, public, \
normalize_timestamp, storage_directory, split_path, validate_sync_to, \
TRUE_VALUES, validate_device_partition
TRUE_VALUES, validate_device_partition, json
from swift.common.constraints import CONTAINER_LISTING_LIMIT, \
check_mount, check_float, check_utf8, FORMAT2CONTENT_TYPE
from swift.common.bufferedhttp import http_connect
@ -362,28 +361,20 @@ class ContainerController(object):
container_list = broker.list_objects_iter(limit, marker, end_marker,
prefix, delimiter, path)
if out_content_type == 'application/json':
json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
'"content_type":%s, "last_modified":"%s"']
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
data = []
for (name, created_at, size, content_type, etag) in container_list:
# escape name and format date here
name = simplejson.dumps(name)
created_at = datetime.utcfromtimestamp(
float(created_at)).isoformat()
# python isoformat() doesn't include msecs when zero
if len(created_at) < len("1970-01-01T00:00:00.000000"):
created_at += ".000000"
if content_type is None:
json_out.append('{"subdir":%s}' % name)
data.append({"subdir": name})
else:
content_type = simplejson.dumps(content_type)
json_out.append(json_pattern % (name,
etag,
size,
content_type,
created_at))
container_list = '[' + ','.join(json_out) + ']'
created_at = datetime.utcfromtimestamp(
float(created_at)).isoformat()
# python isoformat() doesn't include msecs when zero
if len(created_at) < len("1970-01-01T00:00:00.000000"):
created_at += ".000000"
data.append({'last_modified': created_at, 'bytes': size,
'content_type': content_type, 'hash': etag,
'name': name})
container_list = json.dumps(data)
elif out_content_type.endswith('/xml'):
xml_output = []
for (name, created_at, size, content_type, etag) in container_list:
@ -436,7 +427,7 @@ class ContainerController(object):
self.logger.increment('REPLICATE.errors')
return HTTPInsufficientStorage(drive=drive, request=req)
try:
args = simplejson.load(req.environ['wsgi.input'])
args = json.load(req.environ['wsgi.input'])
except ValueError, err:
self.logger.increment('REPLICATE.errors')
return HTTPBadRequest(body=str(err), content_type='text/plain')