openstack/swift
Code Issues Proposed changes

Merge from trunk

Browse Source
This commit is contained in:
gholt 2010-12-30 08:47:52 -08:00
commit 1e34f1afe8
26 changed files with 132 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/st
View File

@ -44,8 +44,6 @@ except:
try:
from swift.common.bufferedhttp \
import BufferedHTTPConnection as HTTPConnection
import gettext
gettext.install('swift', unicode=1)
except:
from httplib import HTTPConnection

2
bin/swift-account-audit
View File

@ -20,7 +20,6 @@ from urllib import quote
from hashlib import md5
import getopt
from itertools import chain
import gettext
import simplejson
from eventlet.greenpool import GreenPool
@ -325,7 +324,6 @@ class Auditor(object):
if __name__ == '__main__':
gettext.install('swift', unicode=1)
try:
optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
except getopt.GetoptError, err:

2
bin/swift-auth-add-user
View File

@ -18,13 +18,11 @@ from ConfigParser import ConfigParser
from optparse import OptionParser
from os.path import basename
from sys import argv, exit
import gettext
from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__':
gettext.install('swift', unicode=1)
default_conf = '/etc/swift/auth-server.conf'
parser = OptionParser(
usage='Usage: %prog [options] <account> <user> <password>')

2
bin/swift-auth-recreate-accounts
View File

@ -17,12 +17,10 @@
from ConfigParser import ConfigParser
from optparse import OptionParser
from sys import argv, exit
import gettext
from swift.common.bufferedhttp import http_connect_raw as http_connect
if __name__ == '__main__':
gettext.install('swift', unicode=1)
default_conf = '/etc/swift/auth-server.conf'
parser = OptionParser(usage='Usage: %prog [options]')
parser.add_option('-c', '--conf', dest='conf', default=default_conf,

2
bin/swift-auth-update-reseller-prefixes
View File

@ -16,13 +16,11 @@
from os.path import basename
from sys import argv, exit
import gettext
from swift.common.db import get_db_connection
if __name__ == '__main__':
gettext.install('swift', unicode=1)
app = basename(argv[0])
if len(argv) != 3:
exit('''

2
bin/swift-bench
View File

@ -20,7 +20,6 @@ import sys
import signal
import uuid
from optparse import OptionParser
import gettext
from swift.common.bench import BenchController
from swift.common.utils import readconf, NamedLogger
@ -56,7 +55,6 @@ SAIO_DEFAULTS = {
}
if __name__ == '__main__':
gettext.install('swift', unicode=1)
usage = "usage: %prog [OPTIONS] [CONF_FILE]"
usage += """\n\nConf file with SAIO defaults:

2
bin/swift-drive-audit
View File

@ -20,7 +20,6 @@ import re
import subprocess
import sys
from ConfigParser import ConfigParser
import gettext
from swift.common.utils import get_logger
@ -87,7 +86,6 @@ def comment_fstab(mount_point):
os.rename('/etc/fstab.new', '/etc/fstab')
if __name__ == '__main__':
gettext.install('swift', unicode=1)
c = ConfigParser()
try:
conf_path = sys.argv[1]

3
bin/swift-get-nodes
View File

@ -16,14 +16,11 @@
import sys
import urllib
import gettext
from swift.common.ring import Ring
from swift.common.utils import hash_path
gettext.install('swift', unicode=1)
if len(sys.argv) < 3 or len(sys.argv) > 5:
print 'Usage: %s <ring.gz> <account> [<container>] [<object>]' \
% sys.argv[0]

2
bin/swift-log-uploader
View File

@ -15,14 +15,12 @@
# limitations under the License.
import sys
import gettext
from swift.stats.log_uploader import LogUploader
from swift.common.utils import parse_options
from swift.common import utils
if __name__ == '__main__':
gettext.install('swift', unicode=1)
conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
try:
plugin = options['extra_args'][0]

2
bin/swift-object-info
View File

@ -18,14 +18,12 @@ import sys
import cPickle as pickle
from datetime import datetime
from hashlib import md5
import gettext
from swift.common.ring import Ring
from swift.obj.server import read_metadata
from swift.common.utils import hash_path
if __name__ == '__main__':
gettext.install('swift', unicode=1)
if len(sys.argv) <= 1:
print "Usage: %s OBJECT_FILE" % sys.argv[0]
sys.exit(1)

2
bin/swift-ring-builder
View File

@ -21,7 +21,6 @@ from os import mkdir
from os.path import basename, dirname, exists, join as pathjoin
from sys import argv, exit
from time import time
import gettext
from swift.common.ring import RingBuilder
@ -175,7 +174,6 @@ swift-ring-builder <builder_file> set_min_part_hours <hours>
if __name__ == '__main__':
gettext.install('swift', unicode=1)
if len(argv) < 2:
print '''
swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s

2
bin/swift-stats-populate
View File

@ -21,7 +21,6 @@ from optparse import OptionParser
from sys import exit, argv
from time import time
from uuid import uuid4
import gettext
from eventlet import GreenPool, patcher, sleep
from eventlet.pools import Pool
@ -76,7 +75,6 @@ def report(success):
if __name__ == '__main__':
global begun, created, item_type, next_report, need_to_create, retries_done
gettext.install('swift', unicode=1)
patcher.monkey_patch()
parser = OptionParser()

2
bin/swift-stats-report
View File

@ -23,7 +23,6 @@ from optparse import OptionParser
from sys import argv, exit, stderr
from time import time
from uuid import uuid4
import gettext
from eventlet import GreenPool, hubs, patcher, sleep, Timeout
from eventlet.pools import Pool
@ -747,7 +746,6 @@ def object_delete_report(coropool, connpool, options):
if __name__ == '__main__':
gettext.install('swift', unicode=1)
patcher.monkey_patch()
hubs.get_hub().debug_exceptions = False

4
swift/__init__.py
View File

@ -1 +1,5 @@
import gettext
__version__ = '1.1.0'
gettext.install('swift')

2
swift/common/daemon.py
View File

@ -16,7 +16,6 @@
import os
import sys
import signal
import gettext
from re import sub
from swift.common import utils
@ -42,7 +41,6 @@ class Daemon(object):
utils.validate_configuration()
utils.capture_stdio(self.logger, **kwargs)
utils.drop_privileges(self.conf.get('user', 'swift'))
gettext.install('swift', unicode=1)
def kill_children(*args):
signal.signal(signal.SIGTERM, signal.SIG_IGN)

1
swift/common/db.py
View File

@ -139,6 +139,7 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
conn.execute('PRAGMA synchronous = NORMAL')
conn.execute('PRAGMA count_changes = OFF')
conn.execute('PRAGMA temp_store = MEMORY')
conn.execute('PRAGMA journal_mode = DELETE')
conn.create_function('chexor', 3, chexor)
except sqlite3.DatabaseError:
import traceback

2
swift/common/middleware/cname_lookup.py
View File

@ -61,7 +61,7 @@ class CNAMELookupMiddleware(object):
port = ''
if ':' in given_domain:
given_domain, port = given_domain.rsplit(':', 1)
if given_domain == self.storage_domain[1:]: # strip initial '.'
if given_domain == self.storage_domain[1:]: # strip initial '.'
return self.app(env, start_response)
a_domain = given_domain
if not a_domain.endswith(self.storage_domain):

2
swift/common/middleware/memcache.py
View File

@ -35,6 +35,8 @@ class MemcacheMiddleware(object):
def filter_factory(global_conf, **local_conf):
conf = global_conf.copy()
conf.update(local_conf)
def cache_filter(app):
return MemcacheMiddleware(app, conf)
return cache_filter

7
swift/common/wsgi.py
View File

@ -21,7 +21,6 @@ import signal
import sys
import time
import mimetools
import gettext
import eventlet
from eventlet import greenio, GreenPool, sleep, wsgi, listen
@ -57,14 +56,15 @@ def monkey_patch_mimetools():
mimetools.Message.parsetype = parsetype
def get_socket(conf, default_port=8080):
"""Bind socket to bind ip:port in conf
:param conf: Configuration dict to read settings from
:param default_port: port to use if not specified in conf
:returns : a socket object as returned from socket.listen or ssl.wrap_socket
if conf specifies cert_file
:returns : a socket object as returned from socket.listen or
ssl.wrap_socket if conf specifies cert_file
"""
bind_addr = (conf.get('bind_ip', '0.0.0.0'),
int(conf.get('bind_port', default_port)))
@ -121,7 +121,6 @@ def run_wsgi(conf_file, app_section, *args, **kwargs):
sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
# remaining tasks should not require elevated privileges
drop_privileges(conf.get('user', 'swift'))
gettext.install('swift', unicode=1)
# finally after binding to ports and privilege drop, run app __init__ code
app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})

2
swift/obj/replicator.py
View File

@ -245,7 +245,7 @@ class ObjectReplicator(Daemon):
except Timeout:
self.logger.error(_("Killing long-running rsync: %s"), str(args))
proc.kill()
return 1 # failure response code
return 1 # failure response code
total_time = time.time() - start_time
for result in results.split('\n'):
if result == '':

27
swift/proxy/server.py
View File

@ -96,9 +96,11 @@ def delay_denial(func):
return func(*a, **kw)
return wrapped
def get_account_memcache_key(account):
return 'account/%s' % account
def get_container_memcache_key(account, container):
return 'container/%s/%s' % (account, container)
@ -298,7 +300,8 @@ class Controller(object):
:param additional_info: additional information to log
"""
self.app.logger.exception(
_('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: %(info)s'),
_('ERROR with %(type)s server %(ip)s:%(port)s/%(device)s re: '
'%(info)s'),
{'type': typ, 'ip': node['ip'], 'port': node['port'],
'device': node['device'], 'info': additional_info})
@ -349,7 +352,7 @@ class Controller(object):
if result_code == 200:
return partition, nodes
elif result_code == 404:
return None, None
return None, None
result_code = 0
attempts_left = self.app.account_ring.replica_count
path = '/%s' % account
@ -1149,19 +1152,17 @@ class ObjectController(Controller):
return HTTPPreconditionFailed(request=req,
body='Destination header must be of the form '
'<container name>/<object name>')
new_source = '/' + self.container_name + '/' + self.object_name
source = '/' + self.container_name + '/' + self.object_name
self.container_name = dest_container
self.object_name = dest_object
new_headers = {}
for k, v in req.headers.items():
new_headers[k] = v
new_headers['X-Copy-From'] = new_source
new_headers['Content-Length'] = 0
del new_headers['Destination']
new_path = '/' + self.account_name + dest
new_req = Request.blank(new_path, environ=req.environ,
headers=new_headers)
return self.PUT(new_req)
# re-write the existing request as a PUT instead of creating a new one
# since this one is already attached to the posthooklogger
req.method = 'PUT'
req.path_info = '/' + self.account_name + dest
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = source
del req.headers['Destination']
return self.PUT(req)
class ContainerController(Controller):

6
swift/stats/access_processor.py
View File

@ -127,7 +127,8 @@ class AccessLogProcessor(object):
d['code'] = int(d['code'])
return d
def process(self, obj_stream, account, container, object_name):
def process(self, obj_stream, data_object_account, data_object_container,
data_object_name):
'''generate hourly groupings of data from one access log file'''
hourly_aggr_info = {}
total_lines = 0
@ -191,7 +192,8 @@ class AccessLogProcessor(object):
hourly_aggr_info[aggr_key] = d
if bad_lines > (total_lines * self.warn_percent):
name = '/'.join([account, container, object_name])
name = '/'.join([data_object_account, data_object_container,
data_object_name])
self.logger.warning('I found a bunch of bad lines in %s '\
'(%d bad, %d total)' % (name, bad_lines, total_lines))
return hourly_aggr_info

5
swift/stats/stats_processor.py
View File

@ -22,10 +22,11 @@ class StatsLogProcessor(object):
def __init__(self, conf):
self.logger = get_logger(conf)
def process(self, obj_stream, account, container, object_name):
def process(self, obj_stream, data_object_account, data_object_container,
data_object_name):
'''generate hourly groupings of data from one stats log file'''
account_totals = {}
year, month, day, hour, _ = object_name.split('/')
year, month, day, hour, _ = data_object_name.split('/')
for line in obj_stream:
if not line:
continue

4
test/functionalnosetests/test_container.py
View File

@ -115,6 +115,10 @@ class TestContainer(unittest.TestCase):
resp.read()
self.assert_(resp.status in (200, 204), resp.status)
self.assertEquals(resp.getheader('x-container-meta-test'), 'Value')
resp = retry(delete, name)
resp.read()
self.assertEquals(resp.status, 204)
name = uuid4().hex
resp = retry(put, name, '')
resp.read()

95
test/functionalnosetests/test_object.py
View File

@ -38,21 +38,110 @@ class TestObject(unittest.TestCase):
if skip:
raise SkipTest
def delete(url, token, parsed, conn, obj):
conn.request('DELETE',
'%s/%s/%s' % (parsed.path, self.container, obj),
'', {'X-Auth-Token': token})
return check_response(conn)
# get list of objects in container
def list(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, self.container),
'', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(list)
object_listing = resp.read()
self.assertEquals(resp.status, 200)
# iterate over object listing and delete all objects
for obj in object_listing.splitlines():
resp = retry(delete, obj)
resp.read()
self.assertEquals(resp.status, 204)
# delete the container
def delete(url, token, parsed, conn):
conn.request('DELETE', '%s/%s/%s' % (parsed.path, self.container,
self.obj), '', {'X-Auth-Token': token})
conn.request('DELETE', parsed.path + '/' + self.container, '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
def test_copy_object(self):
if skip:
raise SkipTest
source = '%s/%s' % (self.container, self.obj)
dest = '%s/%s' % (self.container, 'test_copy')
# get contents of source
def get_source(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, source),
'', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get_source)
source_contents = resp.read()
self.assertEquals(resp.status, 200)
self.assertEquals(source_contents, 'test')
# copy source to dest with X-Copy-From
def put(url, token, parsed, conn):
conn.request('PUT', '%s/%s' % (parsed.path, dest), '',
{'X-Auth-Token': token,
'Content-Length': '0',
'X-Copy-From': source})
return check_response(conn)
resp = retry(put)
contents = resp.read()
self.assertEquals(resp.status, 201)
# contents of dest should be the same as source
def get_dest(url, token, parsed, conn):
conn.request('GET',
'%s/%s' % (parsed.path, dest),
'', {'X-Auth-Token': token})
return check_response(conn)
resp = retry(get_dest)
dest_contents = resp.read()
self.assertEquals(resp.status, 200)
self.assertEquals(dest_contents, source_contents)
# delete the copy
def delete(url, token, parsed, conn):
conn.request('DELETE', parsed.path + '/' + self.container, '',
conn.request('DELETE', '%s/%s' % (parsed.path, dest), '',
{'X-Auth-Token': token})
return check_response(conn)
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
# verify dest does not exist
resp = retry(get_dest)
resp.read()
self.assertEquals(resp.status, 404)
# copy source to dest with COPY
def copy(url, token, parsed, conn):
conn.request('COPY', '%s/%s' % (parsed.path, source), '',
{'X-Auth-Token': token,
'Destination': dest})
return check_response(conn)
resp = retry(copy)
contents = resp.read()
self.assertEquals(resp.status, 201)
# contents of dest should be the same as source
resp = retry(get_dest)
dest_contents = resp.read()
self.assertEquals(resp.status, 200)
self.assertEquals(dest_contents, source_contents)
# delete the copy
resp = retry(delete)
resp.read()
self.assertEquals(resp.status, 204)
def test_public_object(self):
if skip:

2
test/unit/container/test_updater.py
View File

@ -23,6 +23,7 @@ from shutil import rmtree
from eventlet import spawn, TimeoutError, listen
from eventlet.timeout import Timeout
from swift.common import utils
from swift.container import updater as container_updater
from swift.container import server as container_server
from swift.common.db import ContainerBroker
@ -33,6 +34,7 @@ from swift.common.utils import normalize_timestamp
class TestContainerUpdater(unittest.TestCase):
def setUp(self):
utils.HASH_PATH_SUFFIX = 'endcap'
self.path_to_test_xfs = os.environ.get('PATH_TO_TEST_XFS')
if not self.path_to_test_xfs or \
not os.path.exists(self.path_to_test_xfs):